LXR linux/drivers/gpu/drm/radeon/si.c

   1/*
   2 * Copyright 2011 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24
  25#include <linux/firmware.h>
  26#include <linux/module.h>
  27#include <linux/pci.h>
  28#include <linux/slab.h>
  29
  30#include <drm/drm_vblank.h>
  31#include <drm/radeon_drm.h>
  32
  33#include "atom.h"
  34#include "clearstate_si.h"
  35#include "evergreen.h"
  36#include "r600.h"
  37#include "radeon.h"
  38#include "radeon_asic.h"
  39#include "radeon_audio.h"
  40#include "radeon_ucode.h"
  41#include "si_blit_shaders.h"
  42#include "si.h"
  43#include "sid.h"
  44
  45
  46MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
  47MODULE_FIRMWARE("radeon/TAHITI_me.bin");
  48MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
  49MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
  50MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
  51MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
  52MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
  53
  54MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
  55MODULE_FIRMWARE("radeon/tahiti_me.bin");
  56MODULE_FIRMWARE("radeon/tahiti_ce.bin");
  57MODULE_FIRMWARE("radeon/tahiti_mc.bin");
  58MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
  59MODULE_FIRMWARE("radeon/tahiti_smc.bin");
  60
  61MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
  62MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
  63MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
  64MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
  65MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
  66MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
  67MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
  68
  69MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
  70MODULE_FIRMWARE("radeon/pitcairn_me.bin");
  71MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
  72MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
  73MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
  74MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
  75MODULE_FIRMWARE("radeon/pitcairn_k_smc.bin");
  76
  77MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
  78MODULE_FIRMWARE("radeon/VERDE_me.bin");
  79MODULE_FIRMWARE("radeon/VERDE_ce.bin");
  80MODULE_FIRMWARE("radeon/VERDE_mc.bin");
  81MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
  82MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
  83MODULE_FIRMWARE("radeon/VERDE_smc.bin");
  84
  85MODULE_FIRMWARE("radeon/verde_pfp.bin");
  86MODULE_FIRMWARE("radeon/verde_me.bin");
  87MODULE_FIRMWARE("radeon/verde_ce.bin");
  88MODULE_FIRMWARE("radeon/verde_mc.bin");
  89MODULE_FIRMWARE("radeon/verde_rlc.bin");
  90MODULE_FIRMWARE("radeon/verde_smc.bin");
  91MODULE_FIRMWARE("radeon/verde_k_smc.bin");
  92
  93MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
  94MODULE_FIRMWARE("radeon/OLAND_me.bin");
  95MODULE_FIRMWARE("radeon/OLAND_ce.bin");
  96MODULE_FIRMWARE("radeon/OLAND_mc.bin");
  97MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
  98MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
  99MODULE_FIRMWARE("radeon/OLAND_smc.bin");
 100
 101MODULE_FIRMWARE("radeon/oland_pfp.bin");
 102MODULE_FIRMWARE("radeon/oland_me.bin");
 103MODULE_FIRMWARE("radeon/oland_ce.bin");
 104MODULE_FIRMWARE("radeon/oland_mc.bin");
 105MODULE_FIRMWARE("radeon/oland_rlc.bin");
 106MODULE_FIRMWARE("radeon/oland_smc.bin");
 107MODULE_FIRMWARE("radeon/oland_k_smc.bin");
 108
 109MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
 110MODULE_FIRMWARE("radeon/HAINAN_me.bin");
 111MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
 112MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
 113MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
 114MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
 115MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
 116
 117MODULE_FIRMWARE("radeon/hainan_pfp.bin");
 118MODULE_FIRMWARE("radeon/hainan_me.bin");
 119MODULE_FIRMWARE("radeon/hainan_ce.bin");
 120MODULE_FIRMWARE("radeon/hainan_mc.bin");
 121MODULE_FIRMWARE("radeon/hainan_rlc.bin");
 122MODULE_FIRMWARE("radeon/hainan_smc.bin");
 123MODULE_FIRMWARE("radeon/hainan_k_smc.bin");
 124MODULE_FIRMWARE("radeon/banks_k_2_smc.bin");
 125
 126MODULE_FIRMWARE("radeon/si58_mc.bin");
 127
 128static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 129static void si_pcie_gen3_enable(struct radeon_device *rdev);
 130static void si_program_aspm(struct radeon_device *rdev);
 131extern void sumo_rlc_fini(struct radeon_device *rdev);
 132extern int sumo_rlc_init(struct radeon_device *rdev);
 133static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
 134                                         bool enable);
 135static void si_init_pg(struct radeon_device *rdev);
 136static void si_init_cg(struct radeon_device *rdev);
 137static void si_fini_pg(struct radeon_device *rdev);
 138static void si_fini_cg(struct radeon_device *rdev);
 139static void si_rlc_stop(struct radeon_device *rdev);
 140
 141static const u32 crtc_offsets[] =
 142{
 143        EVERGREEN_CRTC0_REGISTER_OFFSET,
 144        EVERGREEN_CRTC1_REGISTER_OFFSET,
 145        EVERGREEN_CRTC2_REGISTER_OFFSET,
 146        EVERGREEN_CRTC3_REGISTER_OFFSET,
 147        EVERGREEN_CRTC4_REGISTER_OFFSET,
 148        EVERGREEN_CRTC5_REGISTER_OFFSET
 149};
 150
 151static const u32 si_disp_int_status[] =
 152{
 153        DISP_INTERRUPT_STATUS,
 154        DISP_INTERRUPT_STATUS_CONTINUE,
 155        DISP_INTERRUPT_STATUS_CONTINUE2,
 156        DISP_INTERRUPT_STATUS_CONTINUE3,
 157        DISP_INTERRUPT_STATUS_CONTINUE4,
 158        DISP_INTERRUPT_STATUS_CONTINUE5
 159};
 160
 161#define DC_HPDx_CONTROL(x)        (DC_HPD1_CONTROL     + (x * 0xc))
 162#define DC_HPDx_INT_CONTROL(x)    (DC_HPD1_INT_CONTROL + (x * 0xc))
 163#define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS  + (x * 0xc))
 164
 165static const u32 verde_rlc_save_restore_register_list[] =
 166{
 167        (0x8000 << 16) | (0x98f4 >> 2),
 168        0x00000000,
 169        (0x8040 << 16) | (0x98f4 >> 2),
 170        0x00000000,
 171        (0x8000 << 16) | (0xe80 >> 2),
 172        0x00000000,
 173        (0x8040 << 16) | (0xe80 >> 2),
 174        0x00000000,
 175        (0x8000 << 16) | (0x89bc >> 2),
 176        0x00000000,
 177        (0x8040 << 16) | (0x89bc >> 2),
 178        0x00000000,
 179        (0x8000 << 16) | (0x8c1c >> 2),
 180        0x00000000,
 181        (0x8040 << 16) | (0x8c1c >> 2),
 182        0x00000000,
 183        (0x9c00 << 16) | (0x98f0 >> 2),
 184        0x00000000,
 185        (0x9c00 << 16) | (0xe7c >> 2),
 186        0x00000000,
 187        (0x8000 << 16) | (0x9148 >> 2),
 188        0x00000000,
 189        (0x8040 << 16) | (0x9148 >> 2),
 190        0x00000000,
 191        (0x9c00 << 16) | (0x9150 >> 2),
 192        0x00000000,
 193        (0x9c00 << 16) | (0x897c >> 2),
 194        0x00000000,
 195        (0x9c00 << 16) | (0x8d8c >> 2),
 196        0x00000000,
 197        (0x9c00 << 16) | (0xac54 >> 2),
 198        0X00000000,
 199        0x3,
 200        (0x9c00 << 16) | (0x98f8 >> 2),
 201        0x00000000,
 202        (0x9c00 << 16) | (0x9910 >> 2),
 203        0x00000000,
 204        (0x9c00 << 16) | (0x9914 >> 2),
 205        0x00000000,
 206        (0x9c00 << 16) | (0x9918 >> 2),
 207        0x00000000,
 208        (0x9c00 << 16) | (0x991c >> 2),
 209        0x00000000,
 210        (0x9c00 << 16) | (0x9920 >> 2),
 211        0x00000000,
 212        (0x9c00 << 16) | (0x9924 >> 2),
 213        0x00000000,
 214        (0x9c00 << 16) | (0x9928 >> 2),
 215        0x00000000,
 216        (0x9c00 << 16) | (0x992c >> 2),
 217        0x00000000,
 218        (0x9c00 << 16) | (0x9930 >> 2),
 219        0x00000000,
 220        (0x9c00 << 16) | (0x9934 >> 2),
 221        0x00000000,
 222        (0x9c00 << 16) | (0x9938 >> 2),
 223        0x00000000,
 224        (0x9c00 << 16) | (0x993c >> 2),
 225        0x00000000,
 226        (0x9c00 << 16) | (0x9940 >> 2),
 227        0x00000000,
 228        (0x9c00 << 16) | (0x9944 >> 2),
 229        0x00000000,
 230        (0x9c00 << 16) | (0x9948 >> 2),
 231        0x00000000,
 232        (0x9c00 << 16) | (0x994c >> 2),
 233        0x00000000,
 234        (0x9c00 << 16) | (0x9950 >> 2),
 235        0x00000000,
 236        (0x9c00 << 16) | (0x9954 >> 2),
 237        0x00000000,
 238        (0x9c00 << 16) | (0x9958 >> 2),
 239        0x00000000,
 240        (0x9c00 << 16) | (0x995c >> 2),
 241        0x00000000,
 242        (0x9c00 << 16) | (0x9960 >> 2),
 243        0x00000000,
 244        (0x9c00 << 16) | (0x9964 >> 2),
 245        0x00000000,
 246        (0x9c00 << 16) | (0x9968 >> 2),
 247        0x00000000,
 248        (0x9c00 << 16) | (0x996c >> 2),
 249        0x00000000,
 250        (0x9c00 << 16) | (0x9970 >> 2),
 251        0x00000000,
 252        (0x9c00 << 16) | (0x9974 >> 2),
 253        0x00000000,
 254        (0x9c00 << 16) | (0x9978 >> 2),
 255        0x00000000,
 256        (0x9c00 << 16) | (0x997c >> 2),
 257        0x00000000,
 258        (0x9c00 << 16) | (0x9980 >> 2),
 259        0x00000000,
 260        (0x9c00 << 16) | (0x9984 >> 2),
 261        0x00000000,
 262        (0x9c00 << 16) | (0x9988 >> 2),
 263        0x00000000,
 264        (0x9c00 << 16) | (0x998c >> 2),
 265        0x00000000,
 266        (0x9c00 << 16) | (0x8c00 >> 2),
 267        0x00000000,
 268        (0x9c00 << 16) | (0x8c14 >> 2),
 269        0x00000000,
 270        (0x9c00 << 16) | (0x8c04 >> 2),
 271        0x00000000,
 272        (0x9c00 << 16) | (0x8c08 >> 2),
 273        0x00000000,
 274        (0x8000 << 16) | (0x9b7c >> 2),
 275        0x00000000,
 276        (0x8040 << 16) | (0x9b7c >> 2),
 277        0x00000000,
 278        (0x8000 << 16) | (0xe84 >> 2),
 279        0x00000000,
 280        (0x8040 << 16) | (0xe84 >> 2),
 281        0x00000000,
 282        (0x8000 << 16) | (0x89c0 >> 2),
 283        0x00000000,
 284        (0x8040 << 16) | (0x89c0 >> 2),
 285        0x00000000,
 286        (0x8000 << 16) | (0x914c >> 2),
 287        0x00000000,
 288        (0x8040 << 16) | (0x914c >> 2),
 289        0x00000000,
 290        (0x8000 << 16) | (0x8c20 >> 2),
 291        0x00000000,
 292        (0x8040 << 16) | (0x8c20 >> 2),
 293        0x00000000,
 294        (0x8000 << 16) | (0x9354 >> 2),
 295        0x00000000,
 296        (0x8040 << 16) | (0x9354 >> 2),
 297        0x00000000,
 298        (0x9c00 << 16) | (0x9060 >> 2),
 299        0x00000000,
 300        (0x9c00 << 16) | (0x9364 >> 2),
 301        0x00000000,
 302        (0x9c00 << 16) | (0x9100 >> 2),
 303        0x00000000,
 304        (0x9c00 << 16) | (0x913c >> 2),
 305        0x00000000,
 306        (0x8000 << 16) | (0x90e0 >> 2),
 307        0x00000000,
 308        (0x8000 << 16) | (0x90e4 >> 2),
 309        0x00000000,
 310        (0x8000 << 16) | (0x90e8 >> 2),
 311        0x00000000,
 312        (0x8040 << 16) | (0x90e0 >> 2),
 313        0x00000000,
 314        (0x8040 << 16) | (0x90e4 >> 2),
 315        0x00000000,
 316        (0x8040 << 16) | (0x90e8 >> 2),
 317        0x00000000,
 318        (0x9c00 << 16) | (0x8bcc >> 2),
 319        0x00000000,
 320        (0x9c00 << 16) | (0x8b24 >> 2),
 321        0x00000000,
 322        (0x9c00 << 16) | (0x88c4 >> 2),
 323        0x00000000,
 324        (0x9c00 << 16) | (0x8e50 >> 2),
 325        0x00000000,
 326        (0x9c00 << 16) | (0x8c0c >> 2),
 327        0x00000000,
 328        (0x9c00 << 16) | (0x8e58 >> 2),
 329        0x00000000,
 330        (0x9c00 << 16) | (0x8e5c >> 2),
 331        0x00000000,
 332        (0x9c00 << 16) | (0x9508 >> 2),
 333        0x00000000,
 334        (0x9c00 << 16) | (0x950c >> 2),
 335        0x00000000,
 336        (0x9c00 << 16) | (0x9494 >> 2),
 337        0x00000000,
 338        (0x9c00 << 16) | (0xac0c >> 2),
 339        0x00000000,
 340        (0x9c00 << 16) | (0xac10 >> 2),
 341        0x00000000,
 342        (0x9c00 << 16) | (0xac14 >> 2),
 343        0x00000000,
 344        (0x9c00 << 16) | (0xae00 >> 2),
 345        0x00000000,
 346        (0x9c00 << 16) | (0xac08 >> 2),
 347        0x00000000,
 348        (0x9c00 << 16) | (0x88d4 >> 2),
 349        0x00000000,
 350        (0x9c00 << 16) | (0x88c8 >> 2),
 351        0x00000000,
 352        (0x9c00 << 16) | (0x88cc >> 2),
 353        0x00000000,
 354        (0x9c00 << 16) | (0x89b0 >> 2),
 355        0x00000000,
 356        (0x9c00 << 16) | (0x8b10 >> 2),
 357        0x00000000,
 358        (0x9c00 << 16) | (0x8a14 >> 2),
 359        0x00000000,
 360        (0x9c00 << 16) | (0x9830 >> 2),
 361        0x00000000,
 362        (0x9c00 << 16) | (0x9834 >> 2),
 363        0x00000000,
 364        (0x9c00 << 16) | (0x9838 >> 2),
 365        0x00000000,
 366        (0x9c00 << 16) | (0x9a10 >> 2),
 367        0x00000000,
 368        (0x8000 << 16) | (0x9870 >> 2),
 369        0x00000000,
 370        (0x8000 << 16) | (0x9874 >> 2),
 371        0x00000000,
 372        (0x8001 << 16) | (0x9870 >> 2),
 373        0x00000000,
 374        (0x8001 << 16) | (0x9874 >> 2),
 375        0x00000000,
 376        (0x8040 << 16) | (0x9870 >> 2),
 377        0x00000000,
 378        (0x8040 << 16) | (0x9874 >> 2),
 379        0x00000000,
 380        (0x8041 << 16) | (0x9870 >> 2),
 381        0x00000000,
 382        (0x8041 << 16) | (0x9874 >> 2),
 383        0x00000000,
 384        0x00000000
 385};
 386
 387static const u32 tahiti_golden_rlc_registers[] =
 388{
 389        0xc424, 0xffffffff, 0x00601005,
 390        0xc47c, 0xffffffff, 0x10104040,
 391        0xc488, 0xffffffff, 0x0100000a,
 392        0xc314, 0xffffffff, 0x00000800,
 393        0xc30c, 0xffffffff, 0x800000f4,
 394        0xf4a8, 0xffffffff, 0x00000000
 395};
 396
 397static const u32 tahiti_golden_registers[] =
 398{
 399        0x9a10, 0x00010000, 0x00018208,
 400        0x9830, 0xffffffff, 0x00000000,
 401        0x9834, 0xf00fffff, 0x00000400,
 402        0x9838, 0x0002021c, 0x00020200,
 403        0xc78, 0x00000080, 0x00000000,
 404        0xd030, 0x000300c0, 0x00800040,
 405        0xd830, 0x000300c0, 0x00800040,
 406        0x5bb0, 0x000000f0, 0x00000070,
 407        0x5bc0, 0x00200000, 0x50100000,
 408        0x7030, 0x31000311, 0x00000011,
 409        0x277c, 0x00000003, 0x000007ff,
 410        0x240c, 0x000007ff, 0x00000000,
 411        0x8a14, 0xf000001f, 0x00000007,
 412        0x8b24, 0xffffffff, 0x00ffffff,
 413        0x8b10, 0x0000ff0f, 0x00000000,
 414        0x28a4c, 0x07ffffff, 0x4e000000,
 415        0x28350, 0x3f3f3fff, 0x2a00126a,
 416        0x30, 0x000000ff, 0x0040,
 417        0x34, 0x00000040, 0x00004040,
 418        0x9100, 0x07ffffff, 0x03000000,
 419        0x8e88, 0x01ff1f3f, 0x00000000,
 420        0x8e84, 0x01ff1f3f, 0x00000000,
 421        0x9060, 0x0000007f, 0x00000020,
 422        0x9508, 0x00010000, 0x00010000,
 423        0xac14, 0x00000200, 0x000002fb,
 424        0xac10, 0xffffffff, 0x0000543b,
 425        0xac0c, 0xffffffff, 0xa9210876,
 426        0x88d0, 0xffffffff, 0x000fff40,
 427        0x88d4, 0x0000001f, 0x00000010,
 428        0x1410, 0x20000000, 0x20fffed8,
 429        0x15c0, 0x000c0fc0, 0x000c0400
 430};
 431
 432static const u32 tahiti_golden_registers2[] =
 433{
 434        0xc64, 0x00000001, 0x00000001
 435};
 436
 437static const u32 pitcairn_golden_rlc_registers[] =
 438{
 439        0xc424, 0xffffffff, 0x00601004,
 440        0xc47c, 0xffffffff, 0x10102020,
 441        0xc488, 0xffffffff, 0x01000020,
 442        0xc314, 0xffffffff, 0x00000800,
 443        0xc30c, 0xffffffff, 0x800000a4
 444};
 445
 446static const u32 pitcairn_golden_registers[] =
 447{
 448        0x9a10, 0x00010000, 0x00018208,
 449        0x9830, 0xffffffff, 0x00000000,
 450        0x9834, 0xf00fffff, 0x00000400,
 451        0x9838, 0x0002021c, 0x00020200,
 452        0xc78, 0x00000080, 0x00000000,
 453        0xd030, 0x000300c0, 0x00800040,
 454        0xd830, 0x000300c0, 0x00800040,
 455        0x5bb0, 0x000000f0, 0x00000070,
 456        0x5bc0, 0x00200000, 0x50100000,
 457        0x7030, 0x31000311, 0x00000011,
 458        0x2ae4, 0x00073ffe, 0x000022a2,
 459        0x240c, 0x000007ff, 0x00000000,
 460        0x8a14, 0xf000001f, 0x00000007,
 461        0x8b24, 0xffffffff, 0x00ffffff,
 462        0x8b10, 0x0000ff0f, 0x00000000,
 463        0x28a4c, 0x07ffffff, 0x4e000000,
 464        0x28350, 0x3f3f3fff, 0x2a00126a,
 465        0x30, 0x000000ff, 0x0040,
 466        0x34, 0x00000040, 0x00004040,
 467        0x9100, 0x07ffffff, 0x03000000,
 468        0x9060, 0x0000007f, 0x00000020,
 469        0x9508, 0x00010000, 0x00010000,
 470        0xac14, 0x000003ff, 0x000000f7,
 471        0xac10, 0xffffffff, 0x00000000,
 472        0xac0c, 0xffffffff, 0x32761054,
 473        0x88d4, 0x0000001f, 0x00000010,
 474        0x15c0, 0x000c0fc0, 0x000c0400
 475};
 476
 477static const u32 verde_golden_rlc_registers[] =
 478{
 479        0xc424, 0xffffffff, 0x033f1005,
 480        0xc47c, 0xffffffff, 0x10808020,
 481        0xc488, 0xffffffff, 0x00800008,
 482        0xc314, 0xffffffff, 0x00001000,
 483        0xc30c, 0xffffffff, 0x80010014
 484};
 485
 486static const u32 verde_golden_registers[] =
 487{
 488        0x9a10, 0x00010000, 0x00018208,
 489        0x9830, 0xffffffff, 0x00000000,
 490        0x9834, 0xf00fffff, 0x00000400,
 491        0x9838, 0x0002021c, 0x00020200,
 492        0xc78, 0x00000080, 0x00000000,
 493        0xd030, 0x000300c0, 0x00800040,
 494        0xd030, 0x000300c0, 0x00800040,
 495        0xd830, 0x000300c0, 0x00800040,
 496        0xd830, 0x000300c0, 0x00800040,
 497        0x5bb0, 0x000000f0, 0x00000070,
 498        0x5bc0, 0x00200000, 0x50100000,
 499        0x7030, 0x31000311, 0x00000011,
 500        0x2ae4, 0x00073ffe, 0x000022a2,
 501        0x2ae4, 0x00073ffe, 0x000022a2,
 502        0x2ae4, 0x00073ffe, 0x000022a2,
 503        0x240c, 0x000007ff, 0x00000000,
 504        0x240c, 0x000007ff, 0x00000000,
 505        0x240c, 0x000007ff, 0x00000000,
 506        0x8a14, 0xf000001f, 0x00000007,
 507        0x8a14, 0xf000001f, 0x00000007,
 508        0x8a14, 0xf000001f, 0x00000007,
 509        0x8b24, 0xffffffff, 0x00ffffff,
 510        0x8b10, 0x0000ff0f, 0x00000000,
 511        0x28a4c, 0x07ffffff, 0x4e000000,
 512        0x28350, 0x3f3f3fff, 0x0000124a,
 513        0x28350, 0x3f3f3fff, 0x0000124a,
 514        0x28350, 0x3f3f3fff, 0x0000124a,
 515        0x30, 0x000000ff, 0x0040,
 516        0x34, 0x00000040, 0x00004040,
 517        0x9100, 0x07ffffff, 0x03000000,
 518        0x9100, 0x07ffffff, 0x03000000,
 519        0x8e88, 0x01ff1f3f, 0x00000000,
 520        0x8e88, 0x01ff1f3f, 0x00000000,
 521        0x8e88, 0x01ff1f3f, 0x00000000,
 522        0x8e84, 0x01ff1f3f, 0x00000000,
 523        0x8e84, 0x01ff1f3f, 0x00000000,
 524        0x8e84, 0x01ff1f3f, 0x00000000,
 525        0x9060, 0x0000007f, 0x00000020,
 526        0x9508, 0x00010000, 0x00010000,
 527        0xac14, 0x000003ff, 0x00000003,
 528        0xac14, 0x000003ff, 0x00000003,
 529        0xac14, 0x000003ff, 0x00000003,
 530        0xac10, 0xffffffff, 0x00000000,
 531        0xac10, 0xffffffff, 0x00000000,
 532        0xac10, 0xffffffff, 0x00000000,
 533        0xac0c, 0xffffffff, 0x00001032,
 534        0xac0c, 0xffffffff, 0x00001032,
 535        0xac0c, 0xffffffff, 0x00001032,
 536        0x88d4, 0x0000001f, 0x00000010,
 537        0x88d4, 0x0000001f, 0x00000010,
 538        0x88d4, 0x0000001f, 0x00000010,
 539        0x15c0, 0x000c0fc0, 0x000c0400
 540};
 541
 542static const u32 oland_golden_rlc_registers[] =
 543{
 544        0xc424, 0xffffffff, 0x00601005,
 545        0xc47c, 0xffffffff, 0x10104040,
 546        0xc488, 0xffffffff, 0x0100000a,
 547        0xc314, 0xffffffff, 0x00000800,
 548        0xc30c, 0xffffffff, 0x800000f4
 549};
 550
 551static const u32 oland_golden_registers[] =
 552{
 553        0x9a10, 0x00010000, 0x00018208,
 554        0x9830, 0xffffffff, 0x00000000,
 555        0x9834, 0xf00fffff, 0x00000400,
 556        0x9838, 0x0002021c, 0x00020200,
 557        0xc78, 0x00000080, 0x00000000,
 558        0xd030, 0x000300c0, 0x00800040,
 559        0xd830, 0x000300c0, 0x00800040,
 560        0x5bb0, 0x000000f0, 0x00000070,
 561        0x5bc0, 0x00200000, 0x50100000,
 562        0x7030, 0x31000311, 0x00000011,
 563        0x2ae4, 0x00073ffe, 0x000022a2,
 564        0x240c, 0x000007ff, 0x00000000,
 565        0x8a14, 0xf000001f, 0x00000007,
 566        0x8b24, 0xffffffff, 0x00ffffff,
 567        0x8b10, 0x0000ff0f, 0x00000000,
 568        0x28a4c, 0x07ffffff, 0x4e000000,
 569        0x28350, 0x3f3f3fff, 0x00000082,
 570        0x30, 0x000000ff, 0x0040,
 571        0x34, 0x00000040, 0x00004040,
 572        0x9100, 0x07ffffff, 0x03000000,
 573        0x9060, 0x0000007f, 0x00000020,
 574        0x9508, 0x00010000, 0x00010000,
 575        0xac14, 0x000003ff, 0x000000f3,
 576        0xac10, 0xffffffff, 0x00000000,
 577        0xac0c, 0xffffffff, 0x00003210,
 578        0x88d4, 0x0000001f, 0x00000010,
 579        0x15c0, 0x000c0fc0, 0x000c0400
 580};
 581
 582static const u32 hainan_golden_registers[] =
 583{
 584        0x9a10, 0x00010000, 0x00018208,
 585        0x9830, 0xffffffff, 0x00000000,
 586        0x9834, 0xf00fffff, 0x00000400,
 587        0x9838, 0x0002021c, 0x00020200,
 588        0xd0c0, 0xff000fff, 0x00000100,
 589        0xd030, 0x000300c0, 0x00800040,
 590        0xd8c0, 0xff000fff, 0x00000100,
 591        0xd830, 0x000300c0, 0x00800040,
 592        0x2ae4, 0x00073ffe, 0x000022a2,
 593        0x240c, 0x000007ff, 0x00000000,
 594        0x8a14, 0xf000001f, 0x00000007,
 595        0x8b24, 0xffffffff, 0x00ffffff,
 596        0x8b10, 0x0000ff0f, 0x00000000,
 597        0x28a4c, 0x07ffffff, 0x4e000000,
 598        0x28350, 0x3f3f3fff, 0x00000000,
 599        0x30, 0x000000ff, 0x0040,
 600        0x34, 0x00000040, 0x00004040,
 601        0x9100, 0x03e00000, 0x03600000,
 602        0x9060, 0x0000007f, 0x00000020,
 603        0x9508, 0x00010000, 0x00010000,
 604        0xac14, 0x000003ff, 0x000000f1,
 605        0xac10, 0xffffffff, 0x00000000,
 606        0xac0c, 0xffffffff, 0x00003210,
 607        0x88d4, 0x0000001f, 0x00000010,
 608        0x15c0, 0x000c0fc0, 0x000c0400
 609};
 610
 611static const u32 hainan_golden_registers2[] =
 612{
 613        0x98f8, 0xffffffff, 0x02010001
 614};
 615
 616static const u32 tahiti_mgcg_cgcg_init[] =
 617{
 618        0xc400, 0xffffffff, 0xfffffffc,
 619        0x802c, 0xffffffff, 0xe0000000,
 620        0x9a60, 0xffffffff, 0x00000100,
 621        0x92a4, 0xffffffff, 0x00000100,
 622        0xc164, 0xffffffff, 0x00000100,
 623        0x9774, 0xffffffff, 0x00000100,
 624        0x8984, 0xffffffff, 0x06000100,
 625        0x8a18, 0xffffffff, 0x00000100,
 626        0x92a0, 0xffffffff, 0x00000100,
 627        0xc380, 0xffffffff, 0x00000100,
 628        0x8b28, 0xffffffff, 0x00000100,
 629        0x9144, 0xffffffff, 0x00000100,
 630        0x8d88, 0xffffffff, 0x00000100,
 631        0x8d8c, 0xffffffff, 0x00000100,
 632        0x9030, 0xffffffff, 0x00000100,
 633        0x9034, 0xffffffff, 0x00000100,
 634        0x9038, 0xffffffff, 0x00000100,
 635        0x903c, 0xffffffff, 0x00000100,
 636        0xad80, 0xffffffff, 0x00000100,
 637        0xac54, 0xffffffff, 0x00000100,
 638        0x897c, 0xffffffff, 0x06000100,
 639        0x9868, 0xffffffff, 0x00000100,
 640        0x9510, 0xffffffff, 0x00000100,
 641        0xaf04, 0xffffffff, 0x00000100,
 642        0xae04, 0xffffffff, 0x00000100,
 643        0x949c, 0xffffffff, 0x00000100,
 644        0x802c, 0xffffffff, 0xe0000000,
 645        0x9160, 0xffffffff, 0x00010000,
 646        0x9164, 0xffffffff, 0x00030002,
 647        0x9168, 0xffffffff, 0x00040007,
 648        0x916c, 0xffffffff, 0x00060005,
 649        0x9170, 0xffffffff, 0x00090008,
 650        0x9174, 0xffffffff, 0x00020001,
 651        0x9178, 0xffffffff, 0x00040003,
 652        0x917c, 0xffffffff, 0x00000007,
 653        0x9180, 0xffffffff, 0x00060005,
 654        0x9184, 0xffffffff, 0x00090008,
 655        0x9188, 0xffffffff, 0x00030002,
 656        0x918c, 0xffffffff, 0x00050004,
 657        0x9190, 0xffffffff, 0x00000008,
 658        0x9194, 0xffffffff, 0x00070006,
 659        0x9198, 0xffffffff, 0x000a0009,
 660        0x919c, 0xffffffff, 0x00040003,
 661        0x91a0, 0xffffffff, 0x00060005,
 662        0x91a4, 0xffffffff, 0x00000009,
 663        0x91a8, 0xffffffff, 0x00080007,
 664        0x91ac, 0xffffffff, 0x000b000a,
 665        0x91b0, 0xffffffff, 0x00050004,
 666        0x91b4, 0xffffffff, 0x00070006,
 667        0x91b8, 0xffffffff, 0x0008000b,
 668        0x91bc, 0xffffffff, 0x000a0009,
 669        0x91c0, 0xffffffff, 0x000d000c,
 670        0x91c4, 0xffffffff, 0x00060005,
 671        0x91c8, 0xffffffff, 0x00080007,
 672        0x91cc, 0xffffffff, 0x0000000b,
 673        0x91d0, 0xffffffff, 0x000a0009,
 674        0x91d4, 0xffffffff, 0x000d000c,
 675        0x91d8, 0xffffffff, 0x00070006,
 676        0x91dc, 0xffffffff, 0x00090008,
 677        0x91e0, 0xffffffff, 0x0000000c,
 678        0x91e4, 0xffffffff, 0x000b000a,
 679        0x91e8, 0xffffffff, 0x000e000d,
 680        0x91ec, 0xffffffff, 0x00080007,
 681        0x91f0, 0xffffffff, 0x000a0009,
 682        0x91f4, 0xffffffff, 0x0000000d,
 683        0x91f8, 0xffffffff, 0x000c000b,
 684        0x91fc, 0xffffffff, 0x000f000e,
 685        0x9200, 0xffffffff, 0x00090008,
 686        0x9204, 0xffffffff, 0x000b000a,
 687        0x9208, 0xffffffff, 0x000c000f,
 688        0x920c, 0xffffffff, 0x000e000d,
 689        0x9210, 0xffffffff, 0x00110010,
 690        0x9214, 0xffffffff, 0x000a0009,
 691        0x9218, 0xffffffff, 0x000c000b,
 692        0x921c, 0xffffffff, 0x0000000f,
 693        0x9220, 0xffffffff, 0x000e000d,
 694        0x9224, 0xffffffff, 0x00110010,
 695        0x9228, 0xffffffff, 0x000b000a,
 696        0x922c, 0xffffffff, 0x000d000c,
 697        0x9230, 0xffffffff, 0x00000010,
 698        0x9234, 0xffffffff, 0x000f000e,
 699        0x9238, 0xffffffff, 0x00120011,
 700        0x923c, 0xffffffff, 0x000c000b,
 701        0x9240, 0xffffffff, 0x000e000d,
 702        0x9244, 0xffffffff, 0x00000011,
 703        0x9248, 0xffffffff, 0x0010000f,
 704        0x924c, 0xffffffff, 0x00130012,
 705        0x9250, 0xffffffff, 0x000d000c,
 706        0x9254, 0xffffffff, 0x000f000e,
 707        0x9258, 0xffffffff, 0x00100013,
 708        0x925c, 0xffffffff, 0x00120011,
 709        0x9260, 0xffffffff, 0x00150014,
 710        0x9264, 0xffffffff, 0x000e000d,
 711        0x9268, 0xffffffff, 0x0010000f,
 712        0x926c, 0xffffffff, 0x00000013,
 713        0x9270, 0xffffffff, 0x00120011,
 714        0x9274, 0xffffffff, 0x00150014,
 715        0x9278, 0xffffffff, 0x000f000e,
 716        0x927c, 0xffffffff, 0x00110010,
 717        0x9280, 0xffffffff, 0x00000014,
 718        0x9284, 0xffffffff, 0x00130012,
 719        0x9288, 0xffffffff, 0x00160015,
 720        0x928c, 0xffffffff, 0x0010000f,
 721        0x9290, 0xffffffff, 0x00120011,
 722        0x9294, 0xffffffff, 0x00000015,
 723        0x9298, 0xffffffff, 0x00140013,
 724        0x929c, 0xffffffff, 0x00170016,
 725        0x9150, 0xffffffff, 0x96940200,
 726        0x8708, 0xffffffff, 0x00900100,
 727        0xc478, 0xffffffff, 0x00000080,
 728        0xc404, 0xffffffff, 0x0020003f,
 729        0x30, 0xffffffff, 0x0000001c,
 730        0x34, 0x000f0000, 0x000f0000,
 731        0x160c, 0xffffffff, 0x00000100,
 732        0x1024, 0xffffffff, 0x00000100,
 733        0x102c, 0x00000101, 0x00000000,
 734        0x20a8, 0xffffffff, 0x00000104,
 735        0x264c, 0x000c0000, 0x000c0000,
 736        0x2648, 0x000c0000, 0x000c0000,
 737        0x55e4, 0xff000fff, 0x00000100,
 738        0x55e8, 0x00000001, 0x00000001,
 739        0x2f50, 0x00000001, 0x00000001,
 740        0x30cc, 0xc0000fff, 0x00000104,
 741        0xc1e4, 0x00000001, 0x00000001,
 742        0xd0c0, 0xfffffff0, 0x00000100,
 743        0xd8c0, 0xfffffff0, 0x00000100
 744};
 745
 746static const u32 pitcairn_mgcg_cgcg_init[] =
 747{
 748        0xc400, 0xffffffff, 0xfffffffc,
 749        0x802c, 0xffffffff, 0xe0000000,
 750        0x9a60, 0xffffffff, 0x00000100,
 751        0x92a4, 0xffffffff, 0x00000100,
 752        0xc164, 0xffffffff, 0x00000100,
 753        0x9774, 0xffffffff, 0x00000100,
 754        0x8984, 0xffffffff, 0x06000100,
 755        0x8a18, 0xffffffff, 0x00000100,
 756        0x92a0, 0xffffffff, 0x00000100,
 757        0xc380, 0xffffffff, 0x00000100,
 758        0x8b28, 0xffffffff, 0x00000100,
 759        0x9144, 0xffffffff, 0x00000100,
 760        0x8d88, 0xffffffff, 0x00000100,
 761        0x8d8c, 0xffffffff, 0x00000100,
 762        0x9030, 0xffffffff, 0x00000100,
 763        0x9034, 0xffffffff, 0x00000100,
 764        0x9038, 0xffffffff, 0x00000100,
 765        0x903c, 0xffffffff, 0x00000100,
 766        0xad80, 0xffffffff, 0x00000100,
 767        0xac54, 0xffffffff, 0x00000100,
 768        0x897c, 0xffffffff, 0x06000100,
 769        0x9868, 0xffffffff, 0x00000100,
 770        0x9510, 0xffffffff, 0x00000100,
 771        0xaf04, 0xffffffff, 0x00000100,
 772        0xae04, 0xffffffff, 0x00000100,
 773        0x949c, 0xffffffff, 0x00000100,
 774        0x802c, 0xffffffff, 0xe0000000,
 775        0x9160, 0xffffffff, 0x00010000,
 776        0x9164, 0xffffffff, 0x00030002,
 777        0x9168, 0xffffffff, 0x00040007,
 778        0x916c, 0xffffffff, 0x00060005,
 779        0x9170, 0xffffffff, 0x00090008,
 780        0x9174, 0xffffffff, 0x00020001,
 781        0x9178, 0xffffffff, 0x00040003,
 782        0x917c, 0xffffffff, 0x00000007,
 783        0x9180, 0xffffffff, 0x00060005,
 784        0x9184, 0xffffffff, 0x00090008,
 785        0x9188, 0xffffffff, 0x00030002,
 786        0x918c, 0xffffffff, 0x00050004,
 787        0x9190, 0xffffffff, 0x00000008,
 788        0x9194, 0xffffffff, 0x00070006,
 789        0x9198, 0xffffffff, 0x000a0009,
 790        0x919c, 0xffffffff, 0x00040003,
 791        0x91a0, 0xffffffff, 0x00060005,
 792        0x91a4, 0xffffffff, 0x00000009,
 793        0x91a8, 0xffffffff, 0x00080007,
 794        0x91ac, 0xffffffff, 0x000b000a,
 795        0x91b0, 0xffffffff, 0x00050004,
 796        0x91b4, 0xffffffff, 0x00070006,
 797        0x91b8, 0xffffffff, 0x0008000b,
 798        0x91bc, 0xffffffff, 0x000a0009,
 799        0x91c0, 0xffffffff, 0x000d000c,
 800        0x9200, 0xffffffff, 0x00090008,
 801        0x9204, 0xffffffff, 0x000b000a,
 802        0x9208, 0xffffffff, 0x000c000f,
 803        0x920c, 0xffffffff, 0x000e000d,
 804        0x9210, 0xffffffff, 0x00110010,
 805        0x9214, 0xffffffff, 0x000a0009,
 806        0x9218, 0xffffffff, 0x000c000b,
 807        0x921c, 0xffffffff, 0x0000000f,
 808        0x9220, 0xffffffff, 0x000e000d,
 809        0x9224, 0xffffffff, 0x00110010,
 810        0x9228, 0xffffffff, 0x000b000a,
 811        0x922c, 0xffffffff, 0x000d000c,
 812        0x9230, 0xffffffff, 0x00000010,
 813        0x9234, 0xffffffff, 0x000f000e,
 814        0x9238, 0xffffffff, 0x00120011,
 815        0x923c, 0xffffffff, 0x000c000b,
 816        0x9240, 0xffffffff, 0x000e000d,
 817        0x9244, 0xffffffff, 0x00000011,
 818        0x9248, 0xffffffff, 0x0010000f,
 819        0x924c, 0xffffffff, 0x00130012,
 820        0x9250, 0xffffffff, 0x000d000c,
 821        0x9254, 0xffffffff, 0x000f000e,
 822        0x9258, 0xffffffff, 0x00100013,
 823        0x925c, 0xffffffff, 0x00120011,
 824        0x9260, 0xffffffff, 0x00150014,
 825        0x9150, 0xffffffff, 0x96940200,
 826        0x8708, 0xffffffff, 0x00900100,
 827        0xc478, 0xffffffff, 0x00000080,
 828        0xc404, 0xffffffff, 0x0020003f,
 829        0x30, 0xffffffff, 0x0000001c,
 830        0x34, 0x000f0000, 0x000f0000,
 831        0x160c, 0xffffffff, 0x00000100,
 832        0x1024, 0xffffffff, 0x00000100,
 833        0x102c, 0x00000101, 0x00000000,
 834        0x20a8, 0xffffffff, 0x00000104,
 835        0x55e4, 0xff000fff, 0x00000100,
 836        0x55e8, 0x00000001, 0x00000001,
 837        0x2f50, 0x00000001, 0x00000001,
 838        0x30cc, 0xc0000fff, 0x00000104,
 839        0xc1e4, 0x00000001, 0x00000001,
 840        0xd0c0, 0xfffffff0, 0x00000100,
 841        0xd8c0, 0xfffffff0, 0x00000100
 842};
 843
 844static const u32 verde_mgcg_cgcg_init[] =
 845{
 846        0xc400, 0xffffffff, 0xfffffffc,
 847        0x802c, 0xffffffff, 0xe0000000,
 848        0x9a60, 0xffffffff, 0x00000100,
 849        0x92a4, 0xffffffff, 0x00000100,
 850        0xc164, 0xffffffff, 0x00000100,
 851        0x9774, 0xffffffff, 0x00000100,
 852        0x8984, 0xffffffff, 0x06000100,
 853        0x8a18, 0xffffffff, 0x00000100,
 854        0x92a0, 0xffffffff, 0x00000100,
 855        0xc380, 0xffffffff, 0x00000100,
 856        0x8b28, 0xffffffff, 0x00000100,
 857        0x9144, 0xffffffff, 0x00000100,
 858        0x8d88, 0xffffffff, 0x00000100,
 859        0x8d8c, 0xffffffff, 0x00000100,
 860        0x9030, 0xffffffff, 0x00000100,
 861        0x9034, 0xffffffff, 0x00000100,
 862        0x9038, 0xffffffff, 0x00000100,
 863        0x903c, 0xffffffff, 0x00000100,
 864        0xad80, 0xffffffff, 0x00000100,
 865        0xac54, 0xffffffff, 0x00000100,
 866        0x897c, 0xffffffff, 0x06000100,
 867        0x9868, 0xffffffff, 0x00000100,
 868        0x9510, 0xffffffff, 0x00000100,
 869        0xaf04, 0xffffffff, 0x00000100,
 870        0xae04, 0xffffffff, 0x00000100,
 871        0x949c, 0xffffffff, 0x00000100,
 872        0x802c, 0xffffffff, 0xe0000000,
 873        0x9160, 0xffffffff, 0x00010000,
 874        0x9164, 0xffffffff, 0x00030002,
 875        0x9168, 0xffffffff, 0x00040007,
 876        0x916c, 0xffffffff, 0x00060005,
 877        0x9170, 0xffffffff, 0x00090008,
 878        0x9174, 0xffffffff, 0x00020001,
 879        0x9178, 0xffffffff, 0x00040003,
 880        0x917c, 0xffffffff, 0x00000007,
 881        0x9180, 0xffffffff, 0x00060005,
 882        0x9184, 0xffffffff, 0x00090008,
 883        0x9188, 0xffffffff, 0x00030002,
 884        0x918c, 0xffffffff, 0x00050004,
 885        0x9190, 0xffffffff, 0x00000008,
 886        0x9194, 0xffffffff, 0x00070006,
 887        0x9198, 0xffffffff, 0x000a0009,
 888        0x919c, 0xffffffff, 0x00040003,
 889        0x91a0, 0xffffffff, 0x00060005,
 890        0x91a4, 0xffffffff, 0x00000009,
 891        0x91a8, 0xffffffff, 0x00080007,
 892        0x91ac, 0xffffffff, 0x000b000a,
 893        0x91b0, 0xffffffff, 0x00050004,
 894        0x91b4, 0xffffffff, 0x00070006,
 895        0x91b8, 0xffffffff, 0x0008000b,
 896        0x91bc, 0xffffffff, 0x000a0009,
 897        0x91c0, 0xffffffff, 0x000d000c,
 898        0x9200, 0xffffffff, 0x00090008,
 899        0x9204, 0xffffffff, 0x000b000a,
 900        0x9208, 0xffffffff, 0x000c000f,
 901        0x920c, 0xffffffff, 0x000e000d,
 902        0x9210, 0xffffffff, 0x00110010,
 903        0x9214, 0xffffffff, 0x000a0009,
 904        0x9218, 0xffffffff, 0x000c000b,
 905        0x921c, 0xffffffff, 0x0000000f,
 906        0x9220, 0xffffffff, 0x000e000d,
 907        0x9224, 0xffffffff, 0x00110010,
 908        0x9228, 0xffffffff, 0x000b000a,
 909        0x922c, 0xffffffff, 0x000d000c,
 910        0x9230, 0xffffffff, 0x00000010,
 911        0x9234, 0xffffffff, 0x000f000e,
 912        0x9238, 0xffffffff, 0x00120011,
 913        0x923c, 0xffffffff, 0x000c000b,
 914        0x9240, 0xffffffff, 0x000e000d,
 915        0x9244, 0xffffffff, 0x00000011,
 916        0x9248, 0xffffffff, 0x0010000f,
 917        0x924c, 0xffffffff, 0x00130012,
 918        0x9250, 0xffffffff, 0x000d000c,
 919        0x9254, 0xffffffff, 0x000f000e,
 920        0x9258, 0xffffffff, 0x00100013,
 921        0x925c, 0xffffffff, 0x00120011,
 922        0x9260, 0xffffffff, 0x00150014,
 923        0x9150, 0xffffffff, 0x96940200,
 924        0x8708, 0xffffffff, 0x00900100,
 925        0xc478, 0xffffffff, 0x00000080,
 926        0xc404, 0xffffffff, 0x0020003f,
 927        0x30, 0xffffffff, 0x0000001c,
 928        0x34, 0x000f0000, 0x000f0000,
 929        0x160c, 0xffffffff, 0x00000100,
 930        0x1024, 0xffffffff, 0x00000100,
 931        0x102c, 0x00000101, 0x00000000,
 932        0x20a8, 0xffffffff, 0x00000104,
 933        0x264c, 0x000c0000, 0x000c0000,
 934        0x2648, 0x000c0000, 0x000c0000,
 935        0x55e4, 0xff000fff, 0x00000100,
 936        0x55e8, 0x00000001, 0x00000001,
 937        0x2f50, 0x00000001, 0x00000001,
 938        0x30cc, 0xc0000fff, 0x00000104,
 939        0xc1e4, 0x00000001, 0x00000001,
 940        0xd0c0, 0xfffffff0, 0x00000100,
 941        0xd8c0, 0xfffffff0, 0x00000100
 942};
 943
 944static const u32 oland_mgcg_cgcg_init[] =
 945{
 946        0xc400, 0xffffffff, 0xfffffffc,
 947        0x802c, 0xffffffff, 0xe0000000,
 948        0x9a60, 0xffffffff, 0x00000100,
 949        0x92a4, 0xffffffff, 0x00000100,
 950        0xc164, 0xffffffff, 0x00000100,
 951        0x9774, 0xffffffff, 0x00000100,
 952        0x8984, 0xffffffff, 0x06000100,
 953        0x8a18, 0xffffffff, 0x00000100,
 954        0x92a0, 0xffffffff, 0x00000100,
 955        0xc380, 0xffffffff, 0x00000100,
 956        0x8b28, 0xffffffff, 0x00000100,
 957        0x9144, 0xffffffff, 0x00000100,
 958        0x8d88, 0xffffffff, 0x00000100,
 959        0x8d8c, 0xffffffff, 0x00000100,
 960        0x9030, 0xffffffff, 0x00000100,
 961        0x9034, 0xffffffff, 0x00000100,
 962        0x9038, 0xffffffff, 0x00000100,
 963        0x903c, 0xffffffff, 0x00000100,
 964        0xad80, 0xffffffff, 0x00000100,
 965        0xac54, 0xffffffff, 0x00000100,
 966        0x897c, 0xffffffff, 0x06000100,
 967        0x9868, 0xffffffff, 0x00000100,
 968        0x9510, 0xffffffff, 0x00000100,
 969        0xaf04, 0xffffffff, 0x00000100,
 970        0xae04, 0xffffffff, 0x00000100,
 971        0x949c, 0xffffffff, 0x00000100,
 972        0x802c, 0xffffffff, 0xe0000000,
 973        0x9160, 0xffffffff, 0x00010000,
 974        0x9164, 0xffffffff, 0x00030002,
 975        0x9168, 0xffffffff, 0x00040007,
 976        0x916c, 0xffffffff, 0x00060005,
 977        0x9170, 0xffffffff, 0x00090008,
 978        0x9174, 0xffffffff, 0x00020001,
 979        0x9178, 0xffffffff, 0x00040003,
 980        0x917c, 0xffffffff, 0x00000007,
 981        0x9180, 0xffffffff, 0x00060005,
 982        0x9184, 0xffffffff, 0x00090008,
 983        0x9188, 0xffffffff, 0x00030002,
 984        0x918c, 0xffffffff, 0x00050004,
 985        0x9190, 0xffffffff, 0x00000008,
 986        0x9194, 0xffffffff, 0x00070006,
 987        0x9198, 0xffffffff, 0x000a0009,
 988        0x919c, 0xffffffff, 0x00040003,
 989        0x91a0, 0xffffffff, 0x00060005,
 990        0x91a4, 0xffffffff, 0x00000009,
 991        0x91a8, 0xffffffff, 0x00080007,
 992        0x91ac, 0xffffffff, 0x000b000a,
 993        0x91b0, 0xffffffff, 0x00050004,
 994        0x91b4, 0xffffffff, 0x00070006,
 995        0x91b8, 0xffffffff, 0x0008000b,
 996        0x91bc, 0xffffffff, 0x000a0009,
 997        0x91c0, 0xffffffff, 0x000d000c,
 998        0x91c4, 0xffffffff, 0x00060005,
 999        0x91c8, 0xffffffff, 0x00080007,
1000        0x91cc, 0xffffffff, 0x0000000b,

1001        0x91d0, 0xffffffff, 0x000a0009,
1002        0x91d4, 0xffffffff, 0x000d000c,
1003        0x9150, 0xffffffff, 0x96940200,
1004        0x8708, 0xffffffff, 0x00900100,
1005        0xc478, 0xffffffff, 0x00000080,
1006        0xc404, 0xffffffff, 0x0020003f,
1007        0x30, 0xffffffff, 0x0000001c,
1008        0x34, 0x000f0000, 0x000f0000,
1009        0x160c, 0xffffffff, 0x00000100,
1010        0x1024, 0xffffffff, 0x00000100,
1011        0x102c, 0x00000101, 0x00000000,
1012        0x20a8, 0xffffffff, 0x00000104,
1013        0x264c, 0x000c0000, 0x000c0000,
1014        0x2648, 0x000c0000, 0x000c0000,
1015        0x55e4, 0xff000fff, 0x00000100,
1016        0x55e8, 0x00000001, 0x00000001,
1017        0x2f50, 0x00000001, 0x00000001,
1018        0x30cc, 0xc0000fff, 0x00000104,
1019        0xc1e4, 0x00000001, 0x00000001,
1020        0xd0c0, 0xfffffff0, 0x00000100,
1021        0xd8c0, 0xfffffff0, 0x00000100
1022};
1023
1024static const u32 hainan_mgcg_cgcg_init[] =
1025{
1026        0xc400, 0xffffffff, 0xfffffffc,
1027        0x802c, 0xffffffff, 0xe0000000,
1028        0x9a60, 0xffffffff, 0x00000100,
1029        0x92a4, 0xffffffff, 0x00000100,
1030        0xc164, 0xffffffff, 0x00000100,
1031        0x9774, 0xffffffff, 0x00000100,
1032        0x8984, 0xffffffff, 0x06000100,
1033        0x8a18, 0xffffffff, 0x00000100,
1034        0x92a0, 0xffffffff, 0x00000100,
1035        0xc380, 0xffffffff, 0x00000100,
1036        0x8b28, 0xffffffff, 0x00000100,
1037        0x9144, 0xffffffff, 0x00000100,
1038        0x8d88, 0xffffffff, 0x00000100,
1039        0x8d8c, 0xffffffff, 0x00000100,
1040        0x9030, 0xffffffff, 0x00000100,
1041        0x9034, 0xffffffff, 0x00000100,
1042        0x9038, 0xffffffff, 0x00000100,
1043        0x903c, 0xffffffff, 0x00000100,
1044        0xad80, 0xffffffff, 0x00000100,
1045        0xac54, 0xffffffff, 0x00000100,
1046        0x897c, 0xffffffff, 0x06000100,
1047        0x9868, 0xffffffff, 0x00000100,
1048        0x9510, 0xffffffff, 0x00000100,
1049        0xaf04, 0xffffffff, 0x00000100,
1050        0xae04, 0xffffffff, 0x00000100,
1051        0x949c, 0xffffffff, 0x00000100,
1052        0x802c, 0xffffffff, 0xe0000000,
1053        0x9160, 0xffffffff, 0x00010000,
1054        0x9164, 0xffffffff, 0x00030002,
1055        0x9168, 0xffffffff, 0x00040007,
1056        0x916c, 0xffffffff, 0x00060005,
1057        0x9170, 0xffffffff, 0x00090008,
1058        0x9174, 0xffffffff, 0x00020001,
1059        0x9178, 0xffffffff, 0x00040003,
1060        0x917c, 0xffffffff, 0x00000007,
1061        0x9180, 0xffffffff, 0x00060005,
1062        0x9184, 0xffffffff, 0x00090008,
1063        0x9188, 0xffffffff, 0x00030002,
1064        0x918c, 0xffffffff, 0x00050004,
1065        0x9190, 0xffffffff, 0x00000008,
1066        0x9194, 0xffffffff, 0x00070006,
1067        0x9198, 0xffffffff, 0x000a0009,
1068        0x919c, 0xffffffff, 0x00040003,
1069        0x91a0, 0xffffffff, 0x00060005,
1070        0x91a4, 0xffffffff, 0x00000009,
1071        0x91a8, 0xffffffff, 0x00080007,
1072        0x91ac, 0xffffffff, 0x000b000a,
1073        0x91b0, 0xffffffff, 0x00050004,
1074        0x91b4, 0xffffffff, 0x00070006,
1075        0x91b8, 0xffffffff, 0x0008000b,
1076        0x91bc, 0xffffffff, 0x000a0009,
1077        0x91c0, 0xffffffff, 0x000d000c,
1078        0x91c4, 0xffffffff, 0x00060005,
1079        0x91c8, 0xffffffff, 0x00080007,
1080        0x91cc, 0xffffffff, 0x0000000b,
1081        0x91d0, 0xffffffff, 0x000a0009,
1082        0x91d4, 0xffffffff, 0x000d000c,
1083        0x9150, 0xffffffff, 0x96940200,
1084        0x8708, 0xffffffff, 0x00900100,
1085        0xc478, 0xffffffff, 0x00000080,
1086        0xc404, 0xffffffff, 0x0020003f,
1087        0x30, 0xffffffff, 0x0000001c,
1088        0x34, 0x000f0000, 0x000f0000,
1089        0x160c, 0xffffffff, 0x00000100,
1090        0x1024, 0xffffffff, 0x00000100,
1091        0x20a8, 0xffffffff, 0x00000104,
1092        0x264c, 0x000c0000, 0x000c0000,
1093        0x2648, 0x000c0000, 0x000c0000,
1094        0x2f50, 0x00000001, 0x00000001,
1095        0x30cc, 0xc0000fff, 0x00000104,
1096        0xc1e4, 0x00000001, 0x00000001,
1097        0xd0c0, 0xfffffff0, 0x00000100,
1098        0xd8c0, 0xfffffff0, 0x00000100
1099};
1100
1101static u32 verde_pg_init[] =
1102{
1103        0x353c, 0xffffffff, 0x40000,
1104        0x3538, 0xffffffff, 0x200010ff,
1105        0x353c, 0xffffffff, 0x0,
1106        0x353c, 0xffffffff, 0x0,
1107        0x353c, 0xffffffff, 0x0,
1108        0x353c, 0xffffffff, 0x0,
1109        0x353c, 0xffffffff, 0x0,
1110        0x353c, 0xffffffff, 0x7007,
1111        0x3538, 0xffffffff, 0x300010ff,
1112        0x353c, 0xffffffff, 0x0,
1113        0x353c, 0xffffffff, 0x0,
1114        0x353c, 0xffffffff, 0x0,
1115        0x353c, 0xffffffff, 0x0,
1116        0x353c, 0xffffffff, 0x0,
1117        0x353c, 0xffffffff, 0x400000,
1118        0x3538, 0xffffffff, 0x100010ff,
1119        0x353c, 0xffffffff, 0x0,
1120        0x353c, 0xffffffff, 0x0,
1121        0x353c, 0xffffffff, 0x0,
1122        0x353c, 0xffffffff, 0x0,
1123        0x353c, 0xffffffff, 0x0,
1124        0x353c, 0xffffffff, 0x120200,
1125        0x3538, 0xffffffff, 0x500010ff,
1126        0x353c, 0xffffffff, 0x0,
1127        0x353c, 0xffffffff, 0x0,
1128        0x353c, 0xffffffff, 0x0,
1129        0x353c, 0xffffffff, 0x0,
1130        0x353c, 0xffffffff, 0x0,
1131        0x353c, 0xffffffff, 0x1e1e16,
1132        0x3538, 0xffffffff, 0x600010ff,
1133        0x353c, 0xffffffff, 0x0,
1134        0x353c, 0xffffffff, 0x0,
1135        0x353c, 0xffffffff, 0x0,
1136        0x353c, 0xffffffff, 0x0,
1137        0x353c, 0xffffffff, 0x0,
1138        0x353c, 0xffffffff, 0x171f1e,
1139        0x3538, 0xffffffff, 0x700010ff,
1140        0x353c, 0xffffffff, 0x0,
1141        0x353c, 0xffffffff, 0x0,
1142        0x353c, 0xffffffff, 0x0,
1143        0x353c, 0xffffffff, 0x0,
1144        0x353c, 0xffffffff, 0x0,
1145        0x353c, 0xffffffff, 0x0,
1146        0x3538, 0xffffffff, 0x9ff,
1147        0x3500, 0xffffffff, 0x0,
1148        0x3504, 0xffffffff, 0x10000800,
1149        0x3504, 0xffffffff, 0xf,
1150        0x3504, 0xffffffff, 0xf,
1151        0x3500, 0xffffffff, 0x4,
1152        0x3504, 0xffffffff, 0x1000051e,
1153        0x3504, 0xffffffff, 0xffff,
1154        0x3504, 0xffffffff, 0xffff,
1155        0x3500, 0xffffffff, 0x8,
1156        0x3504, 0xffffffff, 0x80500,
1157        0x3500, 0xffffffff, 0x12,
1158        0x3504, 0xffffffff, 0x9050c,
1159        0x3500, 0xffffffff, 0x1d,
1160        0x3504, 0xffffffff, 0xb052c,
1161        0x3500, 0xffffffff, 0x2a,
1162        0x3504, 0xffffffff, 0x1053e,
1163        0x3500, 0xffffffff, 0x2d,
1164        0x3504, 0xffffffff, 0x10546,
1165        0x3500, 0xffffffff, 0x30,
1166        0x3504, 0xffffffff, 0xa054e,
1167        0x3500, 0xffffffff, 0x3c,
1168        0x3504, 0xffffffff, 0x1055f,
1169        0x3500, 0xffffffff, 0x3f,
1170        0x3504, 0xffffffff, 0x10567,
1171        0x3500, 0xffffffff, 0x42,
1172        0x3504, 0xffffffff, 0x1056f,
1173        0x3500, 0xffffffff, 0x45,
1174        0x3504, 0xffffffff, 0x10572,
1175        0x3500, 0xffffffff, 0x48,
1176        0x3504, 0xffffffff, 0x20575,
1177        0x3500, 0xffffffff, 0x4c,
1178        0x3504, 0xffffffff, 0x190801,
1179        0x3500, 0xffffffff, 0x67,
1180        0x3504, 0xffffffff, 0x1082a,
1181        0x3500, 0xffffffff, 0x6a,
1182        0x3504, 0xffffffff, 0x1b082d,
1183        0x3500, 0xffffffff, 0x87,
1184        0x3504, 0xffffffff, 0x310851,
1185        0x3500, 0xffffffff, 0xba,
1186        0x3504, 0xffffffff, 0x891,
1187        0x3500, 0xffffffff, 0xbc,
1188        0x3504, 0xffffffff, 0x893,
1189        0x3500, 0xffffffff, 0xbe,
1190        0x3504, 0xffffffff, 0x20895,
1191        0x3500, 0xffffffff, 0xc2,
1192        0x3504, 0xffffffff, 0x20899,
1193        0x3500, 0xffffffff, 0xc6,
1194        0x3504, 0xffffffff, 0x2089d,
1195        0x3500, 0xffffffff, 0xca,
1196        0x3504, 0xffffffff, 0x8a1,
1197        0x3500, 0xffffffff, 0xcc,
1198        0x3504, 0xffffffff, 0x8a3,
1199        0x3500, 0xffffffff, 0xce,
1200        0x3504, 0xffffffff, 0x308a5,
1201        0x3500, 0xffffffff, 0xd3,
1202        0x3504, 0xffffffff, 0x6d08cd,
1203        0x3500, 0xffffffff, 0x142,
1204        0x3504, 0xffffffff, 0x2000095a,
1205        0x3504, 0xffffffff, 0x1,
1206        0x3500, 0xffffffff, 0x144,
1207        0x3504, 0xffffffff, 0x301f095b,
1208        0x3500, 0xffffffff, 0x165,
1209        0x3504, 0xffffffff, 0xc094d,
1210        0x3500, 0xffffffff, 0x173,
1211        0x3504, 0xffffffff, 0xf096d,
1212        0x3500, 0xffffffff, 0x184,
1213        0x3504, 0xffffffff, 0x15097f,
1214        0x3500, 0xffffffff, 0x19b,
1215        0x3504, 0xffffffff, 0xc0998,
1216        0x3500, 0xffffffff, 0x1a9,
1217        0x3504, 0xffffffff, 0x409a7,
1218        0x3500, 0xffffffff, 0x1af,
1219        0x3504, 0xffffffff, 0xcdc,
1220        0x3500, 0xffffffff, 0x1b1,
1221        0x3504, 0xffffffff, 0x800,
1222        0x3508, 0xffffffff, 0x6c9b2000,
1223        0x3510, 0xfc00, 0x2000,
1224        0x3544, 0xffffffff, 0xfc0,
1225        0x28d4, 0x00000100, 0x100
1226};
1227
1228static void si_init_golden_registers(struct radeon_device *rdev)
1229{
1230        switch (rdev->family) {
1231        case CHIP_TAHITI:
1232                radeon_program_register_sequence(rdev,
1233                                                 tahiti_golden_registers,
1234                                                 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1235                radeon_program_register_sequence(rdev,
1236                                                 tahiti_golden_rlc_registers,
1237                                                 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1238                radeon_program_register_sequence(rdev,
1239                                                 tahiti_mgcg_cgcg_init,
1240                                                 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1241                radeon_program_register_sequence(rdev,
1242                                                 tahiti_golden_registers2,
1243                                                 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1244                break;
1245        case CHIP_PITCAIRN:
1246                radeon_program_register_sequence(rdev,
1247                                                 pitcairn_golden_registers,
1248                                                 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1249                radeon_program_register_sequence(rdev,
1250                                                 pitcairn_golden_rlc_registers,
1251                                                 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1252                radeon_program_register_sequence(rdev,
1253                                                 pitcairn_mgcg_cgcg_init,
1254                                                 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1255                break;
1256        case CHIP_VERDE:
1257                radeon_program_register_sequence(rdev,
1258                                                 verde_golden_registers,
1259                                                 (const u32)ARRAY_SIZE(verde_golden_registers));
1260                radeon_program_register_sequence(rdev,
1261                                                 verde_golden_rlc_registers,
1262                                                 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1263                radeon_program_register_sequence(rdev,
1264                                                 verde_mgcg_cgcg_init,
1265                                                 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1266                radeon_program_register_sequence(rdev,
1267                                                 verde_pg_init,
1268                                                 (const u32)ARRAY_SIZE(verde_pg_init));
1269                break;
1270        case CHIP_OLAND:
1271                radeon_program_register_sequence(rdev,
1272                                                 oland_golden_registers,
1273                                                 (const u32)ARRAY_SIZE(oland_golden_registers));
1274                radeon_program_register_sequence(rdev,
1275                                                 oland_golden_rlc_registers,
1276                                                 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1277                radeon_program_register_sequence(rdev,
1278                                                 oland_mgcg_cgcg_init,
1279                                                 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1280                break;
1281        case CHIP_HAINAN:
1282                radeon_program_register_sequence(rdev,
1283                                                 hainan_golden_registers,
1284                                                 (const u32)ARRAY_SIZE(hainan_golden_registers));
1285                radeon_program_register_sequence(rdev,
1286                                                 hainan_golden_registers2,
1287                                                 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1288                radeon_program_register_sequence(rdev,
1289                                                 hainan_mgcg_cgcg_init,
1290                                                 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1291                break;
1292        default:
1293                break;
1294        }
1295}
1296
1297/**
1298 * si_get_allowed_info_register - fetch the register for the info ioctl
1299 *
1300 * @rdev: radeon_device pointer
1301 * @reg: register offset in bytes
1302 * @val: register value
1303 *
1304 * Returns 0 for success or -EINVAL for an invalid register
1305 *
1306 */
1307int si_get_allowed_info_register(struct radeon_device *rdev,
1308                                 u32 reg, u32 *val)
1309{
1310        switch (reg) {
1311        case GRBM_STATUS:
1312        case GRBM_STATUS2:
1313        case GRBM_STATUS_SE0:
1314        case GRBM_STATUS_SE1:
1315        case SRBM_STATUS:
1316        case SRBM_STATUS2:
1317        case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1318        case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1319        case UVD_STATUS:
1320                *val = RREG32(reg);
1321                return 0;
1322        default:
1323                return -EINVAL;
1324        }
1325}
1326
1327#define PCIE_BUS_CLK                10000
1328#define TCLK                        (PCIE_BUS_CLK / 10)
1329
1330/**
1331 * si_get_xclk - get the xclk
1332 *
1333 * @rdev: radeon_device pointer
1334 *
1335 * Returns the reference clock used by the gfx engine
1336 * (SI).
1337 */
1338u32 si_get_xclk(struct radeon_device *rdev)
1339{
1340        u32 reference_clock = rdev->clock.spll.reference_freq;
1341        u32 tmp;
1342
1343        tmp = RREG32(CG_CLKPIN_CNTL_2);
1344        if (tmp & MUX_TCLK_TO_XCLK)
1345                return TCLK;
1346
1347        tmp = RREG32(CG_CLKPIN_CNTL);
1348        if (tmp & XTALIN_DIVIDE)
1349                return reference_clock / 4;
1350
1351        return reference_clock;
1352}
1353
1354/* get temperature in millidegrees */
1355int si_get_temp(struct radeon_device *rdev)
1356{
1357        u32 temp;
1358        int actual_temp = 0;
1359
1360        temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1361                CTF_TEMP_SHIFT;
1362
1363        if (temp & 0x200)
1364                actual_temp = 255;
1365        else
1366                actual_temp = temp & 0x1ff;
1367
1368        actual_temp = (actual_temp * 1000);
1369
1370        return actual_temp;
1371}
1372
1373#define TAHITI_IO_MC_REGS_SIZE 36
1374
1375static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1376        {0x0000006f, 0x03044000},
1377        {0x00000070, 0x0480c018},
1378        {0x00000071, 0x00000040},
1379        {0x00000072, 0x01000000},
1380        {0x00000074, 0x000000ff},
1381        {0x00000075, 0x00143400},
1382        {0x00000076, 0x08ec0800},
1383        {0x00000077, 0x040000cc},
1384        {0x00000079, 0x00000000},
1385        {0x0000007a, 0x21000409},
1386        {0x0000007c, 0x00000000},
1387        {0x0000007d, 0xe8000000},
1388        {0x0000007e, 0x044408a8},
1389        {0x0000007f, 0x00000003},
1390        {0x00000080, 0x00000000},
1391        {0x00000081, 0x01000000},
1392        {0x00000082, 0x02000000},
1393        {0x00000083, 0x00000000},
1394        {0x00000084, 0xe3f3e4f4},
1395        {0x00000085, 0x00052024},
1396        {0x00000087, 0x00000000},
1397        {0x00000088, 0x66036603},
1398        {0x00000089, 0x01000000},
1399        {0x0000008b, 0x1c0a0000},
1400        {0x0000008c, 0xff010000},
1401        {0x0000008e, 0xffffefff},
1402        {0x0000008f, 0xfff3efff},
1403        {0x00000090, 0xfff3efbf},
1404        {0x00000094, 0x00101101},
1405        {0x00000095, 0x00000fff},
1406        {0x00000096, 0x00116fff},
1407        {0x00000097, 0x60010000},
1408        {0x00000098, 0x10010000},
1409        {0x00000099, 0x00006000},
1410        {0x0000009a, 0x00001000},
1411        {0x0000009f, 0x00a77400}
1412};
1413
1414static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1415        {0x0000006f, 0x03044000},
1416        {0x00000070, 0x0480c018},
1417        {0x00000071, 0x00000040},
1418        {0x00000072, 0x01000000},
1419        {0x00000074, 0x000000ff},
1420        {0x00000075, 0x00143400},
1421        {0x00000076, 0x08ec0800},
1422        {0x00000077, 0x040000cc},
1423        {0x00000079, 0x00000000},
1424        {0x0000007a, 0x21000409},
1425        {0x0000007c, 0x00000000},
1426        {0x0000007d, 0xe8000000},
1427        {0x0000007e, 0x044408a8},
1428        {0x0000007f, 0x00000003},
1429        {0x00000080, 0x00000000},
1430        {0x00000081, 0x01000000},
1431        {0x00000082, 0x02000000},
1432        {0x00000083, 0x00000000},
1433        {0x00000084, 0xe3f3e4f4},
1434        {0x00000085, 0x00052024},
1435        {0x00000087, 0x00000000},
1436        {0x00000088, 0x66036603},
1437        {0x00000089, 0x01000000},
1438        {0x0000008b, 0x1c0a0000},
1439        {0x0000008c, 0xff010000},
1440        {0x0000008e, 0xffffefff},
1441        {0x0000008f, 0xfff3efff},
1442        {0x00000090, 0xfff3efbf},
1443        {0x00000094, 0x00101101},
1444        {0x00000095, 0x00000fff},
1445        {0x00000096, 0x00116fff},
1446        {0x00000097, 0x60010000},
1447        {0x00000098, 0x10010000},
1448        {0x00000099, 0x00006000},
1449        {0x0000009a, 0x00001000},
1450        {0x0000009f, 0x00a47400}
1451};
1452
1453static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1454        {0x0000006f, 0x03044000},
1455        {0x00000070, 0x0480c018},
1456        {0x00000071, 0x00000040},
1457        {0x00000072, 0x01000000},
1458        {0x00000074, 0x000000ff},
1459        {0x00000075, 0x00143400},
1460        {0x00000076, 0x08ec0800},
1461        {0x00000077, 0x040000cc},
1462        {0x00000079, 0x00000000},
1463        {0x0000007a, 0x21000409},
1464        {0x0000007c, 0x00000000},
1465        {0x0000007d, 0xe8000000},
1466        {0x0000007e, 0x044408a8},
1467        {0x0000007f, 0x00000003},
1468        {0x00000080, 0x00000000},
1469        {0x00000081, 0x01000000},
1470        {0x00000082, 0x02000000},
1471        {0x00000083, 0x00000000},
1472        {0x00000084, 0xe3f3e4f4},
1473        {0x00000085, 0x00052024},
1474        {0x00000087, 0x00000000},
1475        {0x00000088, 0x66036603},
1476        {0x00000089, 0x01000000},
1477        {0x0000008b, 0x1c0a0000},
1478        {0x0000008c, 0xff010000},
1479        {0x0000008e, 0xffffefff},
1480        {0x0000008f, 0xfff3efff},
1481        {0x00000090, 0xfff3efbf},
1482        {0x00000094, 0x00101101},
1483        {0x00000095, 0x00000fff},
1484        {0x00000096, 0x00116fff},
1485        {0x00000097, 0x60010000},
1486        {0x00000098, 0x10010000},
1487        {0x00000099, 0x00006000},
1488        {0x0000009a, 0x00001000},
1489        {0x0000009f, 0x00a37400}
1490};
1491
1492static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1493        {0x0000006f, 0x03044000},
1494        {0x00000070, 0x0480c018},
1495        {0x00000071, 0x00000040},
1496        {0x00000072, 0x01000000},
1497        {0x00000074, 0x000000ff},
1498        {0x00000075, 0x00143400},
1499        {0x00000076, 0x08ec0800},
1500        {0x00000077, 0x040000cc},
1501        {0x00000079, 0x00000000},
1502        {0x0000007a, 0x21000409},
1503        {0x0000007c, 0x00000000},
1504        {0x0000007d, 0xe8000000},
1505        {0x0000007e, 0x044408a8},
1506        {0x0000007f, 0x00000003},
1507        {0x00000080, 0x00000000},
1508        {0x00000081, 0x01000000},
1509        {0x00000082, 0x02000000},
1510        {0x00000083, 0x00000000},
1511        {0x00000084, 0xe3f3e4f4},
1512        {0x00000085, 0x00052024},
1513        {0x00000087, 0x00000000},
1514        {0x00000088, 0x66036603},
1515        {0x00000089, 0x01000000},
1516        {0x0000008b, 0x1c0a0000},
1517        {0x0000008c, 0xff010000},
1518        {0x0000008e, 0xffffefff},
1519        {0x0000008f, 0xfff3efff},
1520        {0x00000090, 0xfff3efbf},
1521        {0x00000094, 0x00101101},
1522        {0x00000095, 0x00000fff},
1523        {0x00000096, 0x00116fff},
1524        {0x00000097, 0x60010000},
1525        {0x00000098, 0x10010000},
1526        {0x00000099, 0x00006000},
1527        {0x0000009a, 0x00001000},
1528        {0x0000009f, 0x00a17730}
1529};
1530
1531static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1532        {0x0000006f, 0x03044000},
1533        {0x00000070, 0x0480c018},
1534        {0x00000071, 0x00000040},
1535        {0x00000072, 0x01000000},
1536        {0x00000074, 0x000000ff},
1537        {0x00000075, 0x00143400},
1538        {0x00000076, 0x08ec0800},
1539        {0x00000077, 0x040000cc},
1540        {0x00000079, 0x00000000},
1541        {0x0000007a, 0x21000409},
1542        {0x0000007c, 0x00000000},
1543        {0x0000007d, 0xe8000000},
1544        {0x0000007e, 0x044408a8},
1545        {0x0000007f, 0x00000003},
1546        {0x00000080, 0x00000000},
1547        {0x00000081, 0x01000000},
1548        {0x00000082, 0x02000000},
1549        {0x00000083, 0x00000000},
1550        {0x00000084, 0xe3f3e4f4},
1551        {0x00000085, 0x00052024},
1552        {0x00000087, 0x00000000},
1553        {0x00000088, 0x66036603},
1554        {0x00000089, 0x01000000},
1555        {0x0000008b, 0x1c0a0000},
1556        {0x0000008c, 0xff010000},
1557        {0x0000008e, 0xffffefff},
1558        {0x0000008f, 0xfff3efff},
1559        {0x00000090, 0xfff3efbf},
1560        {0x00000094, 0x00101101},
1561        {0x00000095, 0x00000fff},
1562        {0x00000096, 0x00116fff},
1563        {0x00000097, 0x60010000},
1564        {0x00000098, 0x10010000},
1565        {0x00000099, 0x00006000},
1566        {0x0000009a, 0x00001000},
1567        {0x0000009f, 0x00a07730}
1568};
1569
1570/* ucode loading */
1571int si_mc_load_microcode(struct radeon_device *rdev)
1572{
1573        const __be32 *fw_data = NULL;
1574        const __le32 *new_fw_data = NULL;
1575        u32 running;
1576        u32 *io_mc_regs = NULL;
1577        const __le32 *new_io_mc_regs = NULL;
1578        int i, regs_size, ucode_size;
1579
1580        if (!rdev->mc_fw)
1581                return -EINVAL;
1582
1583        if (rdev->new_fw) {
1584                const struct mc_firmware_header_v1_0 *hdr =
1585                        (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1586
1587                radeon_ucode_print_mc_hdr(&hdr->header);
1588                regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1589                new_io_mc_regs = (const __le32 *)
1590                        (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1591                ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1592                new_fw_data = (const __le32 *)
1593                        (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1594        } else {
1595                ucode_size = rdev->mc_fw->size / 4;
1596
1597                switch (rdev->family) {
1598                case CHIP_TAHITI:
1599                        io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1600                        regs_size = TAHITI_IO_MC_REGS_SIZE;
1601                        break;
1602                case CHIP_PITCAIRN:
1603                        io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1604                        regs_size = TAHITI_IO_MC_REGS_SIZE;
1605                        break;
1606                case CHIP_VERDE:
1607                default:
1608                        io_mc_regs = (u32 *)&verde_io_mc_regs;
1609                        regs_size = TAHITI_IO_MC_REGS_SIZE;
1610                        break;
1611                case CHIP_OLAND:
1612                        io_mc_regs = (u32 *)&oland_io_mc_regs;
1613                        regs_size = TAHITI_IO_MC_REGS_SIZE;
1614                        break;
1615                case CHIP_HAINAN:
1616                        io_mc_regs = (u32 *)&hainan_io_mc_regs;
1617                        regs_size = TAHITI_IO_MC_REGS_SIZE;
1618                        break;
1619                }
1620                fw_data = (const __be32 *)rdev->mc_fw->data;
1621        }
1622
1623        running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1624
1625        if (running == 0) {
1626                /* reset the engine and set to writable */
1627                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1628                WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1629
1630                /* load mc io regs */
1631                for (i = 0; i < regs_size; i++) {
1632                        if (rdev->new_fw) {
1633                                WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1634                                WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1635                        } else {
1636                                WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1637                                WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1638                        }
1639                }
1640                /* load the MC ucode */
1641                for (i = 0; i < ucode_size; i++) {
1642                        if (rdev->new_fw)
1643                                WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1644                        else
1645                                WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1646                }
1647
1648                /* put the engine back into the active state */
1649                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1650                WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1651                WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1652
1653                /* wait for training to complete */
1654                for (i = 0; i < rdev->usec_timeout; i++) {
1655                        if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1656                                break;
1657                        udelay(1);
1658                }
1659                for (i = 0; i < rdev->usec_timeout; i++) {
1660                        if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1661                                break;
1662                        udelay(1);
1663                }
1664        }
1665
1666        return 0;
1667}
1668
1669static int si_init_microcode(struct radeon_device *rdev)
1670{
1671        const char *chip_name;
1672        const char *new_chip_name;
1673        size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1674        size_t smc_req_size, mc2_req_size;
1675        char fw_name[30];
1676        int err;
1677        int new_fw = 0;
1678        bool new_smc = false;
1679        bool si58_fw = false;
1680        bool banks2_fw = false;
1681
1682        DRM_DEBUG("\n");
1683
1684        switch (rdev->family) {
1685        case CHIP_TAHITI:
1686                chip_name = "TAHITI";
1687                new_chip_name = "tahiti";
1688                pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1689                me_req_size = SI_PM4_UCODE_SIZE * 4;
1690                ce_req_size = SI_CE_UCODE_SIZE * 4;
1691                rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1692                mc_req_size = SI_MC_UCODE_SIZE * 4;
1693                mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1694                smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1695                break;
1696        case CHIP_PITCAIRN:
1697                chip_name = "PITCAIRN";
1698                if ((rdev->pdev->revision == 0x81) &&
1699                    ((rdev->pdev->device == 0x6810) ||
1700                     (rdev->pdev->device == 0x6811)))
1701                        new_smc = true;
1702                new_chip_name = "pitcairn";
1703                pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1704                me_req_size = SI_PM4_UCODE_SIZE * 4;
1705                ce_req_size = SI_CE_UCODE_SIZE * 4;
1706                rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1707                mc_req_size = SI_MC_UCODE_SIZE * 4;
1708                mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1709                smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1710                break;
1711        case CHIP_VERDE:
1712                chip_name = "VERDE";
1713                if (((rdev->pdev->device == 0x6820) &&
1714                     ((rdev->pdev->revision == 0x81) ||
1715                      (rdev->pdev->revision == 0x83))) ||
1716                    ((rdev->pdev->device == 0x6821) &&
1717                     ((rdev->pdev->revision == 0x83) ||
1718                      (rdev->pdev->revision == 0x87))) ||
1719                    ((rdev->pdev->revision == 0x87) &&
1720                     ((rdev->pdev->device == 0x6823) ||
1721                      (rdev->pdev->device == 0x682b))))
1722                        new_smc = true;
1723                new_chip_name = "verde";
1724                pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1725                me_req_size = SI_PM4_UCODE_SIZE * 4;
1726                ce_req_size = SI_CE_UCODE_SIZE * 4;
1727                rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1728                mc_req_size = SI_MC_UCODE_SIZE * 4;
1729                mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1730                smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1731                break;
1732        case CHIP_OLAND:
1733                chip_name = "OLAND";
1734                if (((rdev->pdev->revision == 0x81) &&
1735                     ((rdev->pdev->device == 0x6600) ||
1736                      (rdev->pdev->device == 0x6604) ||
1737                      (rdev->pdev->device == 0x6605) ||
1738                      (rdev->pdev->device == 0x6610))) ||
1739                    ((rdev->pdev->revision == 0x83) &&
1740                     (rdev->pdev->device == 0x6610)))
1741                        new_smc = true;
1742                new_chip_name = "oland";
1743                pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1744                me_req_size = SI_PM4_UCODE_SIZE * 4;
1745                ce_req_size = SI_CE_UCODE_SIZE * 4;
1746                rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1747                mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1748                smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1749                break;
1750        case CHIP_HAINAN:
1751                chip_name = "HAINAN";
1752                if (((rdev->pdev->revision == 0x81) &&
1753                     (rdev->pdev->device == 0x6660)) ||
1754                    ((rdev->pdev->revision == 0x83) &&
1755                     ((rdev->pdev->device == 0x6660) ||
1756                      (rdev->pdev->device == 0x6663) ||
1757                      (rdev->pdev->device == 0x6665) ||
1758                      (rdev->pdev->device == 0x6667))))
1759                        new_smc = true;
1760                else if ((rdev->pdev->revision == 0xc3) &&
1761                         (rdev->pdev->device == 0x6665))
1762                        banks2_fw = true;
1763                new_chip_name = "hainan";
1764                pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1765                me_req_size = SI_PM4_UCODE_SIZE * 4;
1766                ce_req_size = SI_CE_UCODE_SIZE * 4;
1767                rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1768                mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1769                smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1770                break;
1771        default: BUG();
1772        }
1773
1774        /* this memory configuration requires special firmware */
1775        if (((RREG32(MC_SEQ_MISC0) & 0xff000000) >> 24) == 0x58)
1776                si58_fw = true;
1777
1778        DRM_INFO("Loading %s Microcode\n", new_chip_name);
1779
1780        snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1781        err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1782        if (err) {
1783                snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1784                err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1785                if (err)
1786                        goto out;
1787                if (rdev->pfp_fw->size != pfp_req_size) {
1788                        pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
1789                               rdev->pfp_fw->size, fw_name);
1790                        err = -EINVAL;
1791                        goto out;
1792                }
1793        } else {
1794                err = radeon_ucode_validate(rdev->pfp_fw);
1795                if (err) {
1796                        pr_err("si_cp: validation failed for firmware \"%s\"\n",
1797                               fw_name);
1798                        goto out;
1799                } else {
1800                        new_fw++;
1801                }
1802        }
1803
1804        snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
1805        err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1806        if (err) {
1807                snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1808                err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1809                if (err)
1810                        goto out;
1811                if (rdev->me_fw->size != me_req_size) {
1812                        pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
1813                               rdev->me_fw->size, fw_name);
1814                        err = -EINVAL;
1815                }
1816        } else {
1817                err = radeon_ucode_validate(rdev->me_fw);
1818                if (err) {
1819                        pr_err("si_cp: validation failed for firmware \"%s\"\n",
1820                               fw_name);
1821                        goto out;
1822                } else {
1823                        new_fw++;
1824                }
1825        }
1826
1827        snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
1828        err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1829        if (err) {
1830                snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1831                err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1832                if (err)
1833                        goto out;
1834                if (rdev->ce_fw->size != ce_req_size) {
1835                        pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
1836                               rdev->ce_fw->size, fw_name);
1837                        err = -EINVAL;
1838                }
1839        } else {
1840                err = radeon_ucode_validate(rdev->ce_fw);
1841                if (err) {
1842                        pr_err("si_cp: validation failed for firmware \"%s\"\n",
1843                               fw_name);
1844                        goto out;
1845                } else {
1846                        new_fw++;
1847                }
1848        }
1849
1850        snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
1851        err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1852        if (err) {
1853                snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1854                err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1855                if (err)
1856                        goto out;
1857                if (rdev->rlc_fw->size != rlc_req_size) {
1858                        pr_err("si_rlc: Bogus length %zu in firmware \"%s\"\n",
1859                               rdev->rlc_fw->size, fw_name);
1860                        err = -EINVAL;
1861                }
1862        } else {
1863                err = radeon_ucode_validate(rdev->rlc_fw);
1864                if (err) {
1865                        pr_err("si_cp: validation failed for firmware \"%s\"\n",
1866                               fw_name);
1867                        goto out;
1868                } else {
1869                        new_fw++;
1870                }
1871        }
1872
1873        if (si58_fw)
1874                snprintf(fw_name, sizeof(fw_name), "radeon/si58_mc.bin");
1875        else
1876                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
1877        err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1878        if (err) {
1879                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1880                err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1881                if (err) {
1882                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1883                        err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1884                        if (err)
1885                                goto out;
1886                }
1887                if ((rdev->mc_fw->size != mc_req_size) &&
1888                    (rdev->mc_fw->size != mc2_req_size)) {
1889                        pr_err("si_mc: Bogus length %zu in firmware \"%s\"\n",
1890                               rdev->mc_fw->size, fw_name);
1891                        err = -EINVAL;
1892                }
1893                DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1894        } else {
1895                err = radeon_ucode_validate(rdev->mc_fw);
1896                if (err) {
1897                        pr_err("si_cp: validation failed for firmware \"%s\"\n",
1898                               fw_name);
1899                        goto out;
1900                } else {
1901                        new_fw++;
1902                }
1903        }
1904
1905        if (banks2_fw)
1906                snprintf(fw_name, sizeof(fw_name), "radeon/banks_k_2_smc.bin");
1907        else if (new_smc)
1908                snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
1909        else
1910                snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
1911        err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1912        if (err) {
1913                snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1914                err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1915                if (err) {
1916                        pr_err("smc: error loading firmware \"%s\"\n", fw_name);
1917                        release_firmware(rdev->smc_fw);
1918                        rdev->smc_fw = NULL;
1919                        err = 0;
1920                } else if (rdev->smc_fw->size != smc_req_size) {
1921                        pr_err("si_smc: Bogus length %zu in firmware \"%s\"\n",
1922                               rdev->smc_fw->size, fw_name);
1923                        err = -EINVAL;
1924                }
1925        } else {
1926                err = radeon_ucode_validate(rdev->smc_fw);
1927                if (err) {
1928                        pr_err("si_cp: validation failed for firmware \"%s\"\n",
1929                               fw_name);
1930                        goto out;
1931                } else {
1932                        new_fw++;
1933                }
1934        }
1935
1936        if (new_fw == 0) {
1937                rdev->new_fw = false;
1938        } else if (new_fw < 6) {
1939                pr_err("si_fw: mixing new and old firmware!\n");
1940                err = -EINVAL;
1941        } else {
1942                rdev->new_fw = true;
1943        }
1944out:
1945        if (err) {
1946                if (err != -EINVAL)
1947                        pr_err("si_cp: Failed to load firmware \"%s\"\n",
1948                               fw_name);
1949                release_firmware(rdev->pfp_fw);
1950                rdev->pfp_fw = NULL;
1951                release_firmware(rdev->me_fw);
1952                rdev->me_fw = NULL;
1953                release_firmware(rdev->ce_fw);
1954                rdev->ce_fw = NULL;
1955                release_firmware(rdev->rlc_fw);
1956                rdev->rlc_fw = NULL;
1957                release_firmware(rdev->mc_fw);
1958                rdev->mc_fw = NULL;
1959                release_firmware(rdev->smc_fw);
1960                rdev->smc_fw = NULL;
1961        }
1962        return err;
1963}
1964
1965/* watermark setup */
1966static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1967                                   struct radeon_crtc *radeon_crtc,
1968                                   struct drm_display_mode *mode,
1969                                   struct drm_display_mode *other_mode)
1970{
1971        u32 tmp, buffer_alloc, i;
1972        u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1973        /*
1974         * Line Buffer Setup
1975         * There are 3 line buffers, each one shared by 2 display controllers.
1976         * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1977         * the display controllers.  The paritioning is done via one of four
1978         * preset allocations specified in bits 21:20:
1979         *  0 - half lb
1980         *  2 - whole lb, other crtc must be disabled
1981         */
1982        /* this can get tricky if we have two large displays on a paired group
1983         * of crtcs.  Ideally for multiple large displays we'd assign them to
1984         * non-linked crtcs for maximum line buffer allocation.
1985         */
1986        if (radeon_crtc->base.enabled && mode) {
1987                if (other_mode) {
1988                        tmp = 0; /* 1/2 */
1989                        buffer_alloc = 1;
1990                } else {
1991                        tmp = 2; /* whole */
1992                        buffer_alloc = 2;
1993                }
1994        } else {
1995                tmp = 0;
1996                buffer_alloc = 0;
1997        }
1998
1999        WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
2000               DC_LB_MEMORY_CONFIG(tmp));

2001
2002        WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
2003               DMIF_BUFFERS_ALLOCATED(buffer_alloc));
2004        for (i = 0; i < rdev->usec_timeout; i++) {
2005                if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
2006                    DMIF_BUFFERS_ALLOCATED_COMPLETED)
2007                        break;
2008                udelay(1);
2009        }
2010
2011        if (radeon_crtc->base.enabled && mode) {
2012                switch (tmp) {
2013                case 0:
2014                default:
2015                        return 4096 * 2;
2016                case 2:
2017                        return 8192 * 2;
2018                }
2019        }
2020
2021        /* controller not enabled, so no lb used */
2022        return 0;
2023}
2024
2025static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
2026{
2027        u32 tmp = RREG32(MC_SHARED_CHMAP);
2028
2029        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2030        case 0:
2031        default:
2032                return 1;
2033        case 1:
2034                return 2;
2035        case 2:
2036                return 4;
2037        case 3:
2038                return 8;
2039        case 4:
2040                return 3;
2041        case 5:
2042                return 6;
2043        case 6:
2044                return 10;
2045        case 7:
2046                return 12;
2047        case 8:
2048                return 16;
2049        }
2050}
2051
2052struct dce6_wm_params {
2053        u32 dram_channels; /* number of dram channels */
2054        u32 yclk;          /* bandwidth per dram data pin in kHz */
2055        u32 sclk;          /* engine clock in kHz */
2056        u32 disp_clk;      /* display clock in kHz */
2057        u32 src_width;     /* viewport width */
2058        u32 active_time;   /* active display time in ns */
2059        u32 blank_time;    /* blank time in ns */
2060        bool interlaced;    /* mode is interlaced */
2061        fixed20_12 vsc;    /* vertical scale ratio */
2062        u32 num_heads;     /* number of active crtcs */
2063        u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2064        u32 lb_size;       /* line buffer allocated to pipe */
2065        u32 vtaps;         /* vertical scaler taps */
2066};
2067
2068static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
2069{
2070        /* Calculate raw DRAM Bandwidth */
2071        fixed20_12 dram_efficiency; /* 0.7 */
2072        fixed20_12 yclk, dram_channels, bandwidth;
2073        fixed20_12 a;
2074
2075        a.full = dfixed_const(1000);
2076        yclk.full = dfixed_const(wm->yclk);
2077        yclk.full = dfixed_div(yclk, a);
2078        dram_channels.full = dfixed_const(wm->dram_channels * 4);
2079        a.full = dfixed_const(10);
2080        dram_efficiency.full = dfixed_const(7);
2081        dram_efficiency.full = dfixed_div(dram_efficiency, a);
2082        bandwidth.full = dfixed_mul(dram_channels, yclk);
2083        bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2084
2085        return dfixed_trunc(bandwidth);
2086}
2087
2088static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2089{
2090        /* Calculate DRAM Bandwidth and the part allocated to display. */
2091        fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2092        fixed20_12 yclk, dram_channels, bandwidth;
2093        fixed20_12 a;
2094
2095        a.full = dfixed_const(1000);
2096        yclk.full = dfixed_const(wm->yclk);
2097        yclk.full = dfixed_div(yclk, a);
2098        dram_channels.full = dfixed_const(wm->dram_channels * 4);
2099        a.full = dfixed_const(10);
2100        disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2101        disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2102        bandwidth.full = dfixed_mul(dram_channels, yclk);
2103        bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2104
2105        return dfixed_trunc(bandwidth);
2106}
2107
2108static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2109{
2110        /* Calculate the display Data return Bandwidth */
2111        fixed20_12 return_efficiency; /* 0.8 */
2112        fixed20_12 sclk, bandwidth;
2113        fixed20_12 a;
2114
2115        a.full = dfixed_const(1000);
2116        sclk.full = dfixed_const(wm->sclk);
2117        sclk.full = dfixed_div(sclk, a);
2118        a.full = dfixed_const(10);
2119        return_efficiency.full = dfixed_const(8);
2120        return_efficiency.full = dfixed_div(return_efficiency, a);
2121        a.full = dfixed_const(32);
2122        bandwidth.full = dfixed_mul(a, sclk);
2123        bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2124
2125        return dfixed_trunc(bandwidth);
2126}
2127
2128static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2129{
2130        return 32;
2131}
2132
2133static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2134{
2135        /* Calculate the DMIF Request Bandwidth */
2136        fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2137        fixed20_12 disp_clk, sclk, bandwidth;
2138        fixed20_12 a, b1, b2;
2139        u32 min_bandwidth;
2140
2141        a.full = dfixed_const(1000);
2142        disp_clk.full = dfixed_const(wm->disp_clk);
2143        disp_clk.full = dfixed_div(disp_clk, a);
2144        a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2145        b1.full = dfixed_mul(a, disp_clk);
2146
2147        a.full = dfixed_const(1000);
2148        sclk.full = dfixed_const(wm->sclk);
2149        sclk.full = dfixed_div(sclk, a);
2150        a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2151        b2.full = dfixed_mul(a, sclk);
2152
2153        a.full = dfixed_const(10);
2154        disp_clk_request_efficiency.full = dfixed_const(8);
2155        disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2156
2157        min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2158
2159        a.full = dfixed_const(min_bandwidth);
2160        bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2161
2162        return dfixed_trunc(bandwidth);
2163}
2164
2165static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2166{
2167        /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2168        u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2169        u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2170        u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2171
2172        return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2173}
2174
2175static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2176{
2177        /* Calculate the display mode Average Bandwidth
2178         * DisplayMode should contain the source and destination dimensions,
2179         * timing, etc.
2180         */
2181        fixed20_12 bpp;
2182        fixed20_12 line_time;
2183        fixed20_12 src_width;
2184        fixed20_12 bandwidth;
2185        fixed20_12 a;
2186
2187        a.full = dfixed_const(1000);
2188        line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2189        line_time.full = dfixed_div(line_time, a);
2190        bpp.full = dfixed_const(wm->bytes_per_pixel);
2191        src_width.full = dfixed_const(wm->src_width);
2192        bandwidth.full = dfixed_mul(src_width, bpp);
2193        bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2194        bandwidth.full = dfixed_div(bandwidth, line_time);
2195
2196        return dfixed_trunc(bandwidth);
2197}
2198
2199static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2200{
2201        /* First calcualte the latency in ns */
2202        u32 mc_latency = 2000; /* 2000 ns. */
2203        u32 available_bandwidth = dce6_available_bandwidth(wm);
2204        u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2205        u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2206        u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2207        u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2208                (wm->num_heads * cursor_line_pair_return_time);
2209        u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2210        u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2211        u32 tmp, dmif_size = 12288;
2212        fixed20_12 a, b, c;
2213
2214        if (wm->num_heads == 0)
2215                return 0;
2216
2217        a.full = dfixed_const(2);
2218        b.full = dfixed_const(1);
2219        if ((wm->vsc.full > a.full) ||
2220            ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2221            (wm->vtaps >= 5) ||
2222            ((wm->vsc.full >= a.full) && wm->interlaced))
2223                max_src_lines_per_dst_line = 4;
2224        else
2225                max_src_lines_per_dst_line = 2;
2226
2227        a.full = dfixed_const(available_bandwidth);
2228        b.full = dfixed_const(wm->num_heads);
2229        a.full = dfixed_div(a, b);
2230        tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);
2231        tmp = min(dfixed_trunc(a), tmp);
2232
2233        lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);
2234
2235        a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2236        b.full = dfixed_const(1000);
2237        c.full = dfixed_const(lb_fill_bw);
2238        b.full = dfixed_div(c, b);
2239        a.full = dfixed_div(a, b);
2240        line_fill_time = dfixed_trunc(a);
2241
2242        if (line_fill_time < wm->active_time)
2243                return latency;
2244        else
2245                return latency + (line_fill_time - wm->active_time);
2246
2247}
2248
2249static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2250{
2251        if (dce6_average_bandwidth(wm) <=
2252            (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2253                return true;
2254        else
2255                return false;
2256};
2257
2258static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2259{
2260        if (dce6_average_bandwidth(wm) <=
2261            (dce6_available_bandwidth(wm) / wm->num_heads))
2262                return true;
2263        else
2264                return false;
2265};
2266
2267static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2268{
2269        u32 lb_partitions = wm->lb_size / wm->src_width;
2270        u32 line_time = wm->active_time + wm->blank_time;
2271        u32 latency_tolerant_lines;
2272        u32 latency_hiding;
2273        fixed20_12 a;
2274
2275        a.full = dfixed_const(1);
2276        if (wm->vsc.full > a.full)
2277                latency_tolerant_lines = 1;
2278        else {
2279                if (lb_partitions <= (wm->vtaps + 1))
2280                        latency_tolerant_lines = 1;
2281                else
2282                        latency_tolerant_lines = 2;
2283        }
2284
2285        latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2286
2287        if (dce6_latency_watermark(wm) <= latency_hiding)
2288                return true;
2289        else
2290                return false;
2291}
2292
2293static void dce6_program_watermarks(struct radeon_device *rdev,
2294                                         struct radeon_crtc *radeon_crtc,
2295                                         u32 lb_size, u32 num_heads)
2296{
2297        struct drm_display_mode *mode = &radeon_crtc->base.mode;
2298        struct dce6_wm_params wm_low, wm_high;
2299        u32 dram_channels;
2300        u32 active_time;
2301        u32 line_time = 0;
2302        u32 latency_watermark_a = 0, latency_watermark_b = 0;
2303        u32 priority_a_mark = 0, priority_b_mark = 0;
2304        u32 priority_a_cnt = PRIORITY_OFF;
2305        u32 priority_b_cnt = PRIORITY_OFF;
2306        u32 tmp, arb_control3;
2307        fixed20_12 a, b, c;
2308
2309        if (radeon_crtc->base.enabled && num_heads && mode) {
2310                active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2311                                            (u32)mode->clock);
2312                line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2313                                          (u32)mode->clock);
2314                line_time = min(line_time, (u32)65535);
2315                priority_a_cnt = 0;
2316                priority_b_cnt = 0;
2317
2318                if (rdev->family == CHIP_ARUBA)
2319                        dram_channels = evergreen_get_number_of_dram_channels(rdev);
2320                else
2321                        dram_channels = si_get_number_of_dram_channels(rdev);
2322
2323                /* watermark for high clocks */
2324                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2325                        wm_high.yclk =
2326                                radeon_dpm_get_mclk(rdev, false) * 10;
2327                        wm_high.sclk =
2328                                radeon_dpm_get_sclk(rdev, false) * 10;
2329                } else {
2330                        wm_high.yclk = rdev->pm.current_mclk * 10;
2331                        wm_high.sclk = rdev->pm.current_sclk * 10;
2332                }
2333
2334                wm_high.disp_clk = mode->clock;
2335                wm_high.src_width = mode->crtc_hdisplay;
2336                wm_high.active_time = active_time;
2337                wm_high.blank_time = line_time - wm_high.active_time;
2338                wm_high.interlaced = false;
2339                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2340                        wm_high.interlaced = true;
2341                wm_high.vsc = radeon_crtc->vsc;
2342                wm_high.vtaps = 1;
2343                if (radeon_crtc->rmx_type != RMX_OFF)
2344                        wm_high.vtaps = 2;
2345                wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2346                wm_high.lb_size = lb_size;
2347                wm_high.dram_channels = dram_channels;
2348                wm_high.num_heads = num_heads;
2349
2350                /* watermark for low clocks */
2351                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2352                        wm_low.yclk =
2353                                radeon_dpm_get_mclk(rdev, true) * 10;
2354                        wm_low.sclk =
2355                                radeon_dpm_get_sclk(rdev, true) * 10;
2356                } else {
2357                        wm_low.yclk = rdev->pm.current_mclk * 10;
2358                        wm_low.sclk = rdev->pm.current_sclk * 10;
2359                }
2360
2361                wm_low.disp_clk = mode->clock;
2362                wm_low.src_width = mode->crtc_hdisplay;
2363                wm_low.active_time = active_time;
2364                wm_low.blank_time = line_time - wm_low.active_time;
2365                wm_low.interlaced = false;
2366                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2367                        wm_low.interlaced = true;
2368                wm_low.vsc = radeon_crtc->vsc;
2369                wm_low.vtaps = 1;
2370                if (radeon_crtc->rmx_type != RMX_OFF)
2371                        wm_low.vtaps = 2;
2372                wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2373                wm_low.lb_size = lb_size;
2374                wm_low.dram_channels = dram_channels;
2375                wm_low.num_heads = num_heads;
2376
2377                /* set for high clocks */
2378                latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2379                /* set for low clocks */
2380                latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2381
2382                /* possibly force display priority to high */
2383                /* should really do this at mode validation time... */
2384                if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2385                    !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2386                    !dce6_check_latency_hiding(&wm_high) ||
2387                    (rdev->disp_priority == 2)) {
2388                        DRM_DEBUG_KMS("force priority to high\n");
2389                        priority_a_cnt |= PRIORITY_ALWAYS_ON;
2390                        priority_b_cnt |= PRIORITY_ALWAYS_ON;
2391                }
2392                if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2393                    !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2394                    !dce6_check_latency_hiding(&wm_low) ||
2395                    (rdev->disp_priority == 2)) {
2396                        DRM_DEBUG_KMS("force priority to high\n");
2397                        priority_a_cnt |= PRIORITY_ALWAYS_ON;
2398                        priority_b_cnt |= PRIORITY_ALWAYS_ON;
2399                }
2400
2401                a.full = dfixed_const(1000);
2402                b.full = dfixed_const(mode->clock);
2403                b.full = dfixed_div(b, a);
2404                c.full = dfixed_const(latency_watermark_a);
2405                c.full = dfixed_mul(c, b);
2406                c.full = dfixed_mul(c, radeon_crtc->hsc);
2407                c.full = dfixed_div(c, a);
2408                a.full = dfixed_const(16);
2409                c.full = dfixed_div(c, a);
2410                priority_a_mark = dfixed_trunc(c);
2411                priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2412
2413                a.full = dfixed_const(1000);
2414                b.full = dfixed_const(mode->clock);
2415                b.full = dfixed_div(b, a);
2416                c.full = dfixed_const(latency_watermark_b);
2417                c.full = dfixed_mul(c, b);
2418                c.full = dfixed_mul(c, radeon_crtc->hsc);
2419                c.full = dfixed_div(c, a);
2420                a.full = dfixed_const(16);
2421                c.full = dfixed_div(c, a);
2422                priority_b_mark = dfixed_trunc(c);
2423                priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2424
2425                /* Save number of lines the linebuffer leads before the scanout */
2426                radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2427        }
2428
2429        /* select wm A */
2430        arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2431        tmp = arb_control3;
2432        tmp &= ~LATENCY_WATERMARK_MASK(3);
2433        tmp |= LATENCY_WATERMARK_MASK(1);
2434        WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2435        WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2436               (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2437                LATENCY_HIGH_WATERMARK(line_time)));
2438        /* select wm B */
2439        tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2440        tmp &= ~LATENCY_WATERMARK_MASK(3);
2441        tmp |= LATENCY_WATERMARK_MASK(2);
2442        WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2443        WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2444               (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2445                LATENCY_HIGH_WATERMARK(line_time)));
2446        /* restore original selection */
2447        WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2448
2449        /* write the priority marks */
2450        WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2451        WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2452
2453        /* save values for DPM */
2454        radeon_crtc->line_time = line_time;
2455        radeon_crtc->wm_high = latency_watermark_a;
2456        radeon_crtc->wm_low = latency_watermark_b;
2457}
2458
2459void dce6_bandwidth_update(struct radeon_device *rdev)
2460{
2461        struct drm_display_mode *mode0 = NULL;
2462        struct drm_display_mode *mode1 = NULL;
2463        u32 num_heads = 0, lb_size;
2464        int i;
2465
2466        if (!rdev->mode_info.mode_config_initialized)
2467                return;
2468
2469        radeon_update_display_priority(rdev);
2470
2471        for (i = 0; i < rdev->num_crtc; i++) {
2472                if (rdev->mode_info.crtcs[i]->base.enabled)
2473                        num_heads++;
2474        }
2475        for (i = 0; i < rdev->num_crtc; i += 2) {
2476                mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2477                mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2478                lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2479                dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2480                lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2481                dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2482        }
2483}
2484
2485/*
2486 * Core functions
2487 */
2488static void si_tiling_mode_table_init(struct radeon_device *rdev)
2489{
2490        u32 *tile = rdev->config.si.tile_mode_array;
2491        const u32 num_tile_mode_states =
2492                        ARRAY_SIZE(rdev->config.si.tile_mode_array);
2493        u32 reg_offset, split_equal_to_row_size;
2494
2495        switch (rdev->config.si.mem_row_size_in_kb) {
2496        case 1:
2497                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2498                break;
2499        case 2:
2500        default:
2501                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2502                break;
2503        case 4:
2504                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2505                break;
2506        }
2507
2508        for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2509                tile[reg_offset] = 0;
2510
2511        switch(rdev->family) {
2512        case CHIP_TAHITI:
2513        case CHIP_PITCAIRN:
2514                /* non-AA compressed depth or any compressed stencil */
2515                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2516                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2517                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2518                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2519                           NUM_BANKS(ADDR_SURF_16_BANK) |
2520                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2522                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2523                /* 2xAA/4xAA compressed depth only */
2524                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2525                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2526                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2527                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2528                           NUM_BANKS(ADDR_SURF_16_BANK) |
2529                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2530                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2531                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2532                /* 8xAA compressed depth only */
2533                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2534                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2535                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2536                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2537                           NUM_BANKS(ADDR_SURF_16_BANK) |
2538                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2539                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2540                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2541                /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2542                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2543                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2544                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2545                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2546                           NUM_BANKS(ADDR_SURF_16_BANK) |
2547                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2548                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2549                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2550                /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2551                tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2552                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2553                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2554                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2555                           NUM_BANKS(ADDR_SURF_16_BANK) |
2556                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2557                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2558                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2559                /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2560                tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2561                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2562                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2563                           TILE_SPLIT(split_equal_to_row_size) |
2564                           NUM_BANKS(ADDR_SURF_16_BANK) |
2565                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2566                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2567                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2568                /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2569                tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2570                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2571                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2572                           TILE_SPLIT(split_equal_to_row_size) |
2573                           NUM_BANKS(ADDR_SURF_16_BANK) |
2574                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2575                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2576                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2577                /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2578                tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2579                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2580                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2581                           TILE_SPLIT(split_equal_to_row_size) |
2582                           NUM_BANKS(ADDR_SURF_16_BANK) |
2583                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2584                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2585                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2586                /* 1D and 1D Array Surfaces */
2587                tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2588                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2589                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2590                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2591                           NUM_BANKS(ADDR_SURF_16_BANK) |
2592                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2593                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2594                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2595                /* Displayable maps. */
2596                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2597                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2598                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2599                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2600                           NUM_BANKS(ADDR_SURF_16_BANK) |
2601                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2602                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2603                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2604                /* Display 8bpp. */
2605                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2607                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2608                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2609                           NUM_BANKS(ADDR_SURF_16_BANK) |
2610                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2611                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2612                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2613                /* Display 16bpp. */
2614                tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2615                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2616                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2617                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2618                           NUM_BANKS(ADDR_SURF_16_BANK) |
2619                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2621                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2622                /* Display 32bpp. */
2623                tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2624                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2625                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2626                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2627                           NUM_BANKS(ADDR_SURF_16_BANK) |
2628                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2629                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2630                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2631                /* Thin. */
2632                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2633                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2634                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2635                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2636                           NUM_BANKS(ADDR_SURF_16_BANK) |
2637                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2638                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2639                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2640                /* Thin 8 bpp. */
2641                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2643                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2644                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2645                           NUM_BANKS(ADDR_SURF_16_BANK) |
2646                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2647                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2648                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2649                /* Thin 16 bpp. */
2650                tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2651                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2652                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2653                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2654                           NUM_BANKS(ADDR_SURF_16_BANK) |
2655                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2656                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2657                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2658                /* Thin 32 bpp. */
2659                tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2660                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2661                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2662                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2663                           NUM_BANKS(ADDR_SURF_16_BANK) |
2664                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2665                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2666                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2667                /* Thin 64 bpp. */
2668                tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2669                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2670                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2671                           TILE_SPLIT(split_equal_to_row_size) |
2672                           NUM_BANKS(ADDR_SURF_16_BANK) |
2673                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2674                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2675                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2676                /* 8 bpp PRT. */
2677                tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2678                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2679                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2680                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2681                           NUM_BANKS(ADDR_SURF_16_BANK) |
2682                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2683                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2684                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2685                /* 16 bpp PRT */
2686                tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2687                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2688                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2689                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2690                           NUM_BANKS(ADDR_SURF_16_BANK) |
2691                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2692                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2693                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2694                /* 32 bpp PRT */
2695                tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2696                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2697                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2698                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2699                           NUM_BANKS(ADDR_SURF_16_BANK) |
2700                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2701                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2702                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2703                /* 64 bpp PRT */
2704                tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2705                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2706                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2707                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2708                           NUM_BANKS(ADDR_SURF_16_BANK) |
2709                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2710                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2711                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2712                /* 128 bpp PRT */
2713                tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2714                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2715                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2716                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2717                           NUM_BANKS(ADDR_SURF_8_BANK) |
2718                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2719                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2720                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2721
2722                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2723                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2724                break;
2725
2726        case CHIP_VERDE:
2727        case CHIP_OLAND:
2728        case CHIP_HAINAN:
2729                /* non-AA compressed depth or any compressed stencil */
2730                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2731                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2732                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2733                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2734                           NUM_BANKS(ADDR_SURF_16_BANK) |
2735                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2736                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2737                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2738                /* 2xAA/4xAA compressed depth only */
2739                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2740                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2741                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2742                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2743                           NUM_BANKS(ADDR_SURF_16_BANK) |
2744                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2746                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2747                /* 8xAA compressed depth only */
2748                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2749                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2750                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2751                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2752                           NUM_BANKS(ADDR_SURF_16_BANK) |
2753                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2754                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2755                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2756                /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2757                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2758                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2759                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2760                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2761                           NUM_BANKS(ADDR_SURF_16_BANK) |
2762                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2764                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2765                /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2766                tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2767                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2768                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2769                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2770                           NUM_BANKS(ADDR_SURF_16_BANK) |
2771                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2772                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2773                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2774                /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2775                tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2776                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2777                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2778                           TILE_SPLIT(split_equal_to_row_size) |
2779                           NUM_BANKS(ADDR_SURF_16_BANK) |
2780                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2781                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2782                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2783                /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2784                tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2786                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2787                           TILE_SPLIT(split_equal_to_row_size) |
2788                           NUM_BANKS(ADDR_SURF_16_BANK) |
2789                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2790                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2791                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2792                /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2793                tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2794                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2795                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2796                           TILE_SPLIT(split_equal_to_row_size) |
2797                           NUM_BANKS(ADDR_SURF_16_BANK) |
2798                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2799                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2800                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2801                /* 1D and 1D Array Surfaces */
2802                tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2803                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2804                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2805                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2806                           NUM_BANKS(ADDR_SURF_16_BANK) |
2807                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2808                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2809                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2810                /* Displayable maps. */
2811                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2812                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2813                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2815                           NUM_BANKS(ADDR_SURF_16_BANK) |
2816                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2817                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2818                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2819                /* Display 8bpp. */
2820                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2821                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2822                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2823                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2824                           NUM_BANKS(ADDR_SURF_16_BANK) |
2825                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2826                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2827                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2828                /* Display 16bpp. */
2829                tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2830                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2831                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2832                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2833                           NUM_BANKS(ADDR_SURF_16_BANK) |
2834                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2835                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2836                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2837                /* Display 32bpp. */
2838                tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2839                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2840                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2841                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2842                           NUM_BANKS(ADDR_SURF_16_BANK) |
2843                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2844                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2845                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2846                /* Thin. */
2847                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2848                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2849                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2850                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2851                           NUM_BANKS(ADDR_SURF_16_BANK) |
2852                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2853                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2854                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2855                /* Thin 8 bpp. */
2856                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2857                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2858                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2859                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2860                           NUM_BANKS(ADDR_SURF_16_BANK) |
2861                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2862                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2863                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2864                /* Thin 16 bpp. */
2865                tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2866                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2867                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2868                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2869                           NUM_BANKS(ADDR_SURF_16_BANK) |
2870                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2871                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2872                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2873                /* Thin 32 bpp. */
2874                tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2875                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2876                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2877                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2878                           NUM_BANKS(ADDR_SURF_16_BANK) |
2879                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2880                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2881                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2882                /* Thin 64 bpp. */
2883                tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2884                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2885                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2886                           TILE_SPLIT(split_equal_to_row_size) |
2887                           NUM_BANKS(ADDR_SURF_16_BANK) |
2888                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2889                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2890                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2891                /* 8 bpp PRT. */
2892                tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2893                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2894                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2895                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2896                           NUM_BANKS(ADDR_SURF_16_BANK) |
2897                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2898                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2899                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2900                /* 16 bpp PRT */
2901                tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2902                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2903                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2904                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2905                           NUM_BANKS(ADDR_SURF_16_BANK) |
2906                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2907                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2908                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2909                /* 32 bpp PRT */
2910                tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2911                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2912                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2913                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2914                           NUM_BANKS(ADDR_SURF_16_BANK) |
2915                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2916                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2917                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2918                /* 64 bpp PRT */
2919                tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2920                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2921                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2922                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2923                           NUM_BANKS(ADDR_SURF_16_BANK) |
2924                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2925                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2926                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2927                /* 128 bpp PRT */
2928                tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2929                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2930                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2931                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2932                           NUM_BANKS(ADDR_SURF_8_BANK) |
2933                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2934                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2935                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2936
2937                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2938                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2939                break;
2940
2941        default:
2942                DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2943        }
2944}
2945
2946static void si_select_se_sh(struct radeon_device *rdev,
2947                            u32 se_num, u32 sh_num)
2948{
2949        u32 data = INSTANCE_BROADCAST_WRITES;
2950
2951        if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2952                data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2953        else if (se_num == 0xffffffff)
2954                data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2955        else if (sh_num == 0xffffffff)
2956                data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2957        else
2958                data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2959        WREG32(GRBM_GFX_INDEX, data);
2960}
2961
2962static u32 si_create_bitmask(u32 bit_width)
2963{
2964        u32 i, mask = 0;
2965
2966        for (i = 0; i < bit_width; i++) {
2967                mask <<= 1;
2968                mask |= 1;
2969        }
2970        return mask;
2971}
2972
2973static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2974{
2975        u32 data, mask;
2976
2977        data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2978        if (data & 1)
2979                data &= INACTIVE_CUS_MASK;
2980        else
2981                data = 0;
2982        data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2983
2984        data >>= INACTIVE_CUS_SHIFT;
2985
2986        mask = si_create_bitmask(cu_per_sh);
2987
2988        return ~data & mask;
2989}
2990
2991static void si_setup_spi(struct radeon_device *rdev,
2992                         u32 se_num, u32 sh_per_se,
2993                         u32 cu_per_sh)
2994{
2995        int i, j, k;
2996        u32 data, mask, active_cu;
2997
2998        for (i = 0; i < se_num; i++) {
2999                for (j = 0; j < sh_per_se; j++) {
3000                        si_select_se_sh(rdev, i, j);

3001                        data = RREG32(SPI_STATIC_THREAD_MGMT_3);
3002                        active_cu = si_get_cu_enabled(rdev, cu_per_sh);
3003
3004                        mask = 1;
3005                        for (k = 0; k < 16; k++) {
3006                                mask <<= k;
3007                                if (active_cu & mask) {
3008                                        data &= ~mask;
3009                                        WREG32(SPI_STATIC_THREAD_MGMT_3, data);
3010                                        break;
3011                                }
3012                        }
3013                }
3014        }
3015        si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3016}
3017
3018static u32 si_get_rb_disabled(struct radeon_device *rdev,
3019                              u32 max_rb_num_per_se,
3020                              u32 sh_per_se)
3021{
3022        u32 data, mask;
3023
3024        data = RREG32(CC_RB_BACKEND_DISABLE);
3025        if (data & 1)
3026                data &= BACKEND_DISABLE_MASK;
3027        else
3028                data = 0;
3029        data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3030
3031        data >>= BACKEND_DISABLE_SHIFT;
3032
3033        mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
3034
3035        return data & mask;
3036}
3037
3038static void si_setup_rb(struct radeon_device *rdev,
3039                        u32 se_num, u32 sh_per_se,
3040                        u32 max_rb_num_per_se)
3041{
3042        int i, j;
3043        u32 data, mask;
3044        u32 disabled_rbs = 0;
3045        u32 enabled_rbs = 0;
3046
3047        for (i = 0; i < se_num; i++) {
3048                for (j = 0; j < sh_per_se; j++) {
3049                        si_select_se_sh(rdev, i, j);
3050                        data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3051                        disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3052                }
3053        }
3054        si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3055
3056        mask = 1;
3057        for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3058                if (!(disabled_rbs & mask))
3059                        enabled_rbs |= mask;
3060                mask <<= 1;
3061        }
3062
3063        rdev->config.si.backend_enable_mask = enabled_rbs;
3064
3065        for (i = 0; i < se_num; i++) {
3066                si_select_se_sh(rdev, i, 0xffffffff);
3067                data = 0;
3068                for (j = 0; j < sh_per_se; j++) {
3069                        switch (enabled_rbs & 3) {
3070                        case 1:
3071                                data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3072                                break;
3073                        case 2:
3074                                data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3075                                break;
3076                        case 3:
3077                        default:
3078                                data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3079                                break;
3080                        }
3081                        enabled_rbs >>= 2;
3082                }
3083                WREG32(PA_SC_RASTER_CONFIG, data);
3084        }
3085        si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3086}
3087
3088static void si_gpu_init(struct radeon_device *rdev)
3089{
3090        u32 gb_addr_config = 0;
3091        u32 mc_arb_ramcfg;
3092        u32 sx_debug_1;
3093        u32 hdp_host_path_cntl;
3094        u32 tmp;
3095        int i, j;
3096
3097        switch (rdev->family) {
3098        case CHIP_TAHITI:
3099                rdev->config.si.max_shader_engines = 2;
3100                rdev->config.si.max_tile_pipes = 12;
3101                rdev->config.si.max_cu_per_sh = 8;
3102                rdev->config.si.max_sh_per_se = 2;
3103                rdev->config.si.max_backends_per_se = 4;
3104                rdev->config.si.max_texture_channel_caches = 12;
3105                rdev->config.si.max_gprs = 256;
3106                rdev->config.si.max_gs_threads = 32;
3107                rdev->config.si.max_hw_contexts = 8;
3108
3109                rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3110                rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3111                rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3112                rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3113                gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3114                break;
3115        case CHIP_PITCAIRN:
3116                rdev->config.si.max_shader_engines = 2;
3117                rdev->config.si.max_tile_pipes = 8;
3118                rdev->config.si.max_cu_per_sh = 5;
3119                rdev->config.si.max_sh_per_se = 2;
3120                rdev->config.si.max_backends_per_se = 4;
3121                rdev->config.si.max_texture_channel_caches = 8;
3122                rdev->config.si.max_gprs = 256;
3123                rdev->config.si.max_gs_threads = 32;
3124                rdev->config.si.max_hw_contexts = 8;
3125
3126                rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3127                rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3128                rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3129                rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3130                gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3131                break;
3132        case CHIP_VERDE:
3133        default:
3134                rdev->config.si.max_shader_engines = 1;
3135                rdev->config.si.max_tile_pipes = 4;
3136                rdev->config.si.max_cu_per_sh = 5;
3137                rdev->config.si.max_sh_per_se = 2;
3138                rdev->config.si.max_backends_per_se = 4;
3139                rdev->config.si.max_texture_channel_caches = 4;
3140                rdev->config.si.max_gprs = 256;
3141                rdev->config.si.max_gs_threads = 32;
3142                rdev->config.si.max_hw_contexts = 8;
3143
3144                rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3145                rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3146                rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3147                rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3148                gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3149                break;
3150        case CHIP_OLAND:
3151                rdev->config.si.max_shader_engines = 1;
3152                rdev->config.si.max_tile_pipes = 4;
3153                rdev->config.si.max_cu_per_sh = 6;
3154                rdev->config.si.max_sh_per_se = 1;
3155                rdev->config.si.max_backends_per_se = 2;
3156                rdev->config.si.max_texture_channel_caches = 4;
3157                rdev->config.si.max_gprs = 256;
3158                rdev->config.si.max_gs_threads = 16;
3159                rdev->config.si.max_hw_contexts = 8;
3160
3161                rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3162                rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3163                rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3164                rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3165                gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3166                break;
3167        case CHIP_HAINAN:
3168                rdev->config.si.max_shader_engines = 1;
3169                rdev->config.si.max_tile_pipes = 4;
3170                rdev->config.si.max_cu_per_sh = 5;
3171                rdev->config.si.max_sh_per_se = 1;
3172                rdev->config.si.max_backends_per_se = 1;
3173                rdev->config.si.max_texture_channel_caches = 2;
3174                rdev->config.si.max_gprs = 256;
3175                rdev->config.si.max_gs_threads = 16;
3176                rdev->config.si.max_hw_contexts = 8;
3177
3178                rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3179                rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3180                rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3181                rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3182                gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3183                break;
3184        }
3185
3186        /* Initialize HDP */
3187        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3188                WREG32((0x2c14 + j), 0x00000000);
3189                WREG32((0x2c18 + j), 0x00000000);
3190                WREG32((0x2c1c + j), 0x00000000);
3191                WREG32((0x2c20 + j), 0x00000000);
3192                WREG32((0x2c24 + j), 0x00000000);
3193        }
3194
3195        WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3196        WREG32(SRBM_INT_CNTL, 1);
3197        WREG32(SRBM_INT_ACK, 1);
3198
3199        evergreen_fix_pci_max_read_req_size(rdev);
3200
3201        WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3202
3203        RREG32(MC_SHARED_CHMAP);
3204        mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3205
3206        rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3207        rdev->config.si.mem_max_burst_length_bytes = 256;
3208        tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3209        rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3210        if (rdev->config.si.mem_row_size_in_kb > 4)
3211                rdev->config.si.mem_row_size_in_kb = 4;
3212        /* XXX use MC settings? */
3213        rdev->config.si.shader_engine_tile_size = 32;
3214        rdev->config.si.num_gpus = 1;
3215        rdev->config.si.multi_gpu_tile_size = 64;
3216
3217        /* fix up row size */
3218        gb_addr_config &= ~ROW_SIZE_MASK;
3219        switch (rdev->config.si.mem_row_size_in_kb) {
3220        case 1:
3221        default:
3222                gb_addr_config |= ROW_SIZE(0);
3223                break;
3224        case 2:
3225                gb_addr_config |= ROW_SIZE(1);
3226                break;
3227        case 4:
3228                gb_addr_config |= ROW_SIZE(2);
3229                break;
3230        }
3231
3232        /* setup tiling info dword.  gb_addr_config is not adequate since it does
3233         * not have bank info, so create a custom tiling dword.
3234         * bits 3:0   num_pipes
3235         * bits 7:4   num_banks
3236         * bits 11:8  group_size
3237         * bits 15:12 row_size
3238         */
3239        rdev->config.si.tile_config = 0;
3240        switch (rdev->config.si.num_tile_pipes) {
3241        case 1:
3242                rdev->config.si.tile_config |= (0 << 0);
3243                break;
3244        case 2:
3245                rdev->config.si.tile_config |= (1 << 0);
3246                break;
3247        case 4:
3248                rdev->config.si.tile_config |= (2 << 0);
3249                break;
3250        case 8:
3251        default:
3252                /* XXX what about 12? */
3253                rdev->config.si.tile_config |= (3 << 0);
3254                break;
3255        }
3256        switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3257        case 0: /* four banks */
3258                rdev->config.si.tile_config |= 0 << 4;
3259                break;
3260        case 1: /* eight banks */
3261                rdev->config.si.tile_config |= 1 << 4;
3262                break;
3263        case 2: /* sixteen banks */
3264        default:
3265                rdev->config.si.tile_config |= 2 << 4;
3266                break;
3267        }
3268        rdev->config.si.tile_config |=
3269                ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3270        rdev->config.si.tile_config |=
3271                ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3272
3273        WREG32(GB_ADDR_CONFIG, gb_addr_config);
3274        WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3275        WREG32(DMIF_ADDR_CALC, gb_addr_config);
3276        WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3277        WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3278        WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3279        if (rdev->has_uvd) {
3280                WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3281                WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3282                WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3283        }
3284
3285        si_tiling_mode_table_init(rdev);
3286
3287        si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3288                    rdev->config.si.max_sh_per_se,
3289                    rdev->config.si.max_backends_per_se);
3290
3291        si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3292                     rdev->config.si.max_sh_per_se,
3293                     rdev->config.si.max_cu_per_sh);
3294
3295        rdev->config.si.active_cus = 0;
3296        for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3297                for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3298                        rdev->config.si.active_cus +=
3299                                hweight32(si_get_cu_active_bitmap(rdev, i, j));
3300                }
3301        }
3302
3303        /* set HW defaults for 3D engine */
3304        WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3305                                     ROQ_IB2_START(0x2b)));
3306        WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3307
3308        sx_debug_1 = RREG32(SX_DEBUG_1);
3309        WREG32(SX_DEBUG_1, sx_debug_1);
3310
3311        WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3312
3313        WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3314                                 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3315                                 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3316                                 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3317
3318        WREG32(VGT_NUM_INSTANCES, 1);
3319
3320        WREG32(CP_PERFMON_CNTL, 0);
3321
3322        WREG32(SQ_CONFIG, 0);
3323
3324        WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3325                                          FORCE_EOV_MAX_REZ_CNT(255)));
3326
3327        WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3328               AUTO_INVLD_EN(ES_AND_GS_AUTO));
3329
3330        WREG32(VGT_GS_VERTEX_REUSE, 16);
3331        WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3332
3333        WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3334        WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3335        WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3336        WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3337        WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3338        WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3339        WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3340        WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3341
3342        tmp = RREG32(HDP_MISC_CNTL);
3343        tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3344        WREG32(HDP_MISC_CNTL, tmp);
3345
3346        hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3347        WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3348
3349        WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3350
3351        udelay(50);
3352}
3353
3354/*
3355 * GPU scratch registers helpers function.
3356 */
3357static void si_scratch_init(struct radeon_device *rdev)
3358{
3359        int i;
3360
3361        rdev->scratch.num_reg = 7;
3362        rdev->scratch.reg_base = SCRATCH_REG0;
3363        for (i = 0; i < rdev->scratch.num_reg; i++) {
3364                rdev->scratch.free[i] = true;
3365                rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3366        }
3367}
3368
3369void si_fence_ring_emit(struct radeon_device *rdev,
3370                        struct radeon_fence *fence)
3371{
3372        struct radeon_ring *ring = &rdev->ring[fence->ring];
3373        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3374
3375        /* flush read cache over gart */
3376        radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3377        radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3378        radeon_ring_write(ring, 0);
3379        radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3380        radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3381                          PACKET3_TC_ACTION_ENA |
3382                          PACKET3_SH_KCACHE_ACTION_ENA |
3383                          PACKET3_SH_ICACHE_ACTION_ENA);
3384        radeon_ring_write(ring, 0xFFFFFFFF);
3385        radeon_ring_write(ring, 0);
3386        radeon_ring_write(ring, 10); /* poll interval */
3387        /* EVENT_WRITE_EOP - flush caches, send int */
3388        radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3389        radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3390        radeon_ring_write(ring, lower_32_bits(addr));
3391        radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3392        radeon_ring_write(ring, fence->seq);
3393        radeon_ring_write(ring, 0);
3394}
3395
3396/*
3397 * IB stuff
3398 */
3399void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3400{
3401        struct radeon_ring *ring = &rdev->ring[ib->ring];
3402        unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3403        u32 header;
3404
3405        if (ib->is_const_ib) {
3406                /* set switch buffer packet before const IB */
3407                radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3408                radeon_ring_write(ring, 0);
3409
3410                header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3411        } else {
3412                u32 next_rptr;
3413                if (ring->rptr_save_reg) {
3414                        next_rptr = ring->wptr + 3 + 4 + 8;
3415                        radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3416                        radeon_ring_write(ring, ((ring->rptr_save_reg -
3417                                                  PACKET3_SET_CONFIG_REG_START) >> 2));
3418                        radeon_ring_write(ring, next_rptr);
3419                } else if (rdev->wb.enabled) {
3420                        next_rptr = ring->wptr + 5 + 4 + 8;
3421                        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3422                        radeon_ring_write(ring, (1 << 8));
3423                        radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3424                        radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3425                        radeon_ring_write(ring, next_rptr);
3426                }
3427
3428                header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3429        }
3430
3431        radeon_ring_write(ring, header);
3432        radeon_ring_write(ring,
3433#ifdef __BIG_ENDIAN
3434                          (2 << 0) |
3435#endif
3436                          (ib->gpu_addr & 0xFFFFFFFC));
3437        radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3438        radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
3439
3440        if (!ib->is_const_ib) {
3441                /* flush read cache over gart for this vmid */
3442                radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3443                radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3444                radeon_ring_write(ring, vm_id);
3445                radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3446                radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3447                                  PACKET3_TC_ACTION_ENA |
3448                                  PACKET3_SH_KCACHE_ACTION_ENA |
3449                                  PACKET3_SH_ICACHE_ACTION_ENA);
3450                radeon_ring_write(ring, 0xFFFFFFFF);
3451                radeon_ring_write(ring, 0);
3452                radeon_ring_write(ring, 10); /* poll interval */
3453        }
3454}
3455
3456/*
3457 * CP.
3458 */
3459static void si_cp_enable(struct radeon_device *rdev, bool enable)
3460{
3461        if (enable)
3462                WREG32(CP_ME_CNTL, 0);
3463        else {
3464                if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3465                        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3466                WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3467                WREG32(SCRATCH_UMSK, 0);
3468                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3469                rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3470                rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3471        }
3472        udelay(50);
3473}
3474
3475static int si_cp_load_microcode(struct radeon_device *rdev)
3476{
3477        int i;
3478
3479        if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3480                return -EINVAL;
3481
3482        si_cp_enable(rdev, false);
3483
3484        if (rdev->new_fw) {
3485                const struct gfx_firmware_header_v1_0 *pfp_hdr =
3486                        (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3487                const struct gfx_firmware_header_v1_0 *ce_hdr =
3488                        (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3489                const struct gfx_firmware_header_v1_0 *me_hdr =
3490                        (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3491                const __le32 *fw_data;
3492                u32 fw_size;
3493
3494                radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3495                radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3496                radeon_ucode_print_gfx_hdr(&me_hdr->header);
3497
3498                /* PFP */
3499                fw_data = (const __le32 *)
3500                        (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3501                fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3502                WREG32(CP_PFP_UCODE_ADDR, 0);
3503                for (i = 0; i < fw_size; i++)
3504                        WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3505                WREG32(CP_PFP_UCODE_ADDR, 0);
3506
3507                /* CE */
3508                fw_data = (const __le32 *)
3509                        (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3510                fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3511                WREG32(CP_CE_UCODE_ADDR, 0);
3512                for (i = 0; i < fw_size; i++)
3513                        WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3514                WREG32(CP_CE_UCODE_ADDR, 0);
3515
3516                /* ME */
3517                fw_data = (const __be32 *)
3518                        (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3519                fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3520                WREG32(CP_ME_RAM_WADDR, 0);
3521                for (i = 0; i < fw_size; i++)
3522                        WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3523                WREG32(CP_ME_RAM_WADDR, 0);
3524        } else {
3525                const __be32 *fw_data;
3526
3527                /* PFP */
3528                fw_data = (const __be32 *)rdev->pfp_fw->data;
3529                WREG32(CP_PFP_UCODE_ADDR, 0);
3530                for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3531                        WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3532                WREG32(CP_PFP_UCODE_ADDR, 0);
3533
3534                /* CE */
3535                fw_data = (const __be32 *)rdev->ce_fw->data;
3536                WREG32(CP_CE_UCODE_ADDR, 0);
3537                for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3538                        WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3539                WREG32(CP_CE_UCODE_ADDR, 0);
3540
3541                /* ME */
3542                fw_data = (const __be32 *)rdev->me_fw->data;
3543                WREG32(CP_ME_RAM_WADDR, 0);
3544                for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3545                        WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3546                WREG32(CP_ME_RAM_WADDR, 0);
3547        }
3548
3549        WREG32(CP_PFP_UCODE_ADDR, 0);
3550        WREG32(CP_CE_UCODE_ADDR, 0);
3551        WREG32(CP_ME_RAM_WADDR, 0);
3552        WREG32(CP_ME_RAM_RADDR, 0);
3553        return 0;
3554}
3555
3556static int si_cp_start(struct radeon_device *rdev)
3557{
3558        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3559        int r, i;
3560
3561        r = radeon_ring_lock(rdev, ring, 7 + 4);
3562        if (r) {
3563                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3564                return r;
3565        }
3566        /* init the CP */
3567        radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3568        radeon_ring_write(ring, 0x1);
3569        radeon_ring_write(ring, 0x0);
3570        radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3571        radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3572        radeon_ring_write(ring, 0);
3573        radeon_ring_write(ring, 0);
3574
3575        /* init the CE partitions */
3576        radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3577        radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3578        radeon_ring_write(ring, 0xc000);
3579        radeon_ring_write(ring, 0xe000);
3580        radeon_ring_unlock_commit(rdev, ring, false);
3581
3582        si_cp_enable(rdev, true);
3583
3584        r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3585        if (r) {
3586                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3587                return r;
3588        }
3589
3590        /* setup clear context state */
3591        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3592        radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3593
3594        for (i = 0; i < si_default_size; i++)
3595                radeon_ring_write(ring, si_default_state[i]);
3596
3597        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3598        radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3599
3600        /* set clear context state */
3601        radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3602        radeon_ring_write(ring, 0);
3603
3604        radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3605        radeon_ring_write(ring, 0x00000316);
3606        radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3607        radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3608
3609        radeon_ring_unlock_commit(rdev, ring, false);
3610
3611        for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3612                ring = &rdev->ring[i];
3613                r = radeon_ring_lock(rdev, ring, 2);
3614
3615                /* clear the compute context state */
3616                radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3617                radeon_ring_write(ring, 0);
3618
3619                radeon_ring_unlock_commit(rdev, ring, false);
3620        }
3621
3622        return 0;
3623}
3624
3625static void si_cp_fini(struct radeon_device *rdev)
3626{
3627        struct radeon_ring *ring;
3628        si_cp_enable(rdev, false);
3629
3630        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3631        radeon_ring_fini(rdev, ring);
3632        radeon_scratch_free(rdev, ring->rptr_save_reg);
3633
3634        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3635        radeon_ring_fini(rdev, ring);
3636        radeon_scratch_free(rdev, ring->rptr_save_reg);
3637
3638        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3639        radeon_ring_fini(rdev, ring);
3640        radeon_scratch_free(rdev, ring->rptr_save_reg);
3641}
3642
3643static int si_cp_resume(struct radeon_device *rdev)
3644{
3645        struct radeon_ring *ring;
3646        u32 tmp;
3647        u32 rb_bufsz;
3648        int r;
3649
3650        si_enable_gui_idle_interrupt(rdev, false);
3651
3652        WREG32(CP_SEM_WAIT_TIMER, 0x0);
3653        WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3654
3655        /* Set the write pointer delay */
3656        WREG32(CP_RB_WPTR_DELAY, 0);
3657
3658        WREG32(CP_DEBUG, 0);
3659        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3660
3661        /* ring 0 - compute and gfx */
3662        /* Set ring buffer size */
3663        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3664        rb_bufsz = order_base_2(ring->ring_size / 8);
3665        tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3666#ifdef __BIG_ENDIAN
3667        tmp |= BUF_SWAP_32BIT;
3668#endif
3669        WREG32(CP_RB0_CNTL, tmp);
3670
3671        /* Initialize the ring buffer's read and write pointers */
3672        WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3673        ring->wptr = 0;
3674        WREG32(CP_RB0_WPTR, ring->wptr);
3675
3676        /* set the wb address whether it's enabled or not */
3677        WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3678        WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3679
3680        if (rdev->wb.enabled)
3681                WREG32(SCRATCH_UMSK, 0xff);
3682        else {
3683                tmp |= RB_NO_UPDATE;
3684                WREG32(SCRATCH_UMSK, 0);
3685        }
3686
3687        mdelay(1);
3688        WREG32(CP_RB0_CNTL, tmp);
3689
3690        WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3691
3692        /* ring1  - compute only */
3693        /* Set ring buffer size */
3694        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3695        rb_bufsz = order_base_2(ring->ring_size / 8);
3696        tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3697#ifdef __BIG_ENDIAN
3698        tmp |= BUF_SWAP_32BIT;
3699#endif
3700        WREG32(CP_RB1_CNTL, tmp);
3701
3702        /* Initialize the ring buffer's read and write pointers */
3703        WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3704        ring->wptr = 0;
3705        WREG32(CP_RB1_WPTR, ring->wptr);
3706
3707        /* set the wb address whether it's enabled or not */
3708        WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3709        WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3710
3711        mdelay(1);
3712        WREG32(CP_RB1_CNTL, tmp);
3713
3714        WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3715
3716        /* ring2 - compute only */
3717        /* Set ring buffer size */
3718        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3719        rb_bufsz = order_base_2(ring->ring_size / 8);
3720        tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3721#ifdef __BIG_ENDIAN
3722        tmp |= BUF_SWAP_32BIT;
3723#endif
3724        WREG32(CP_RB2_CNTL, tmp);
3725
3726        /* Initialize the ring buffer's read and write pointers */
3727        WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3728        ring->wptr = 0;
3729        WREG32(CP_RB2_WPTR, ring->wptr);
3730
3731        /* set the wb address whether it's enabled or not */
3732        WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3733        WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3734
3735        mdelay(1);
3736        WREG32(CP_RB2_CNTL, tmp);
3737
3738        WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3739
3740        /* start the rings */
3741        si_cp_start(rdev);
3742        rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3743        rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3744        rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3745        r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3746        if (r) {
3747                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3748                rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3749                rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3750                return r;
3751        }
3752        r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3753        if (r) {
3754                rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3755        }
3756        r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3757        if (r) {
3758                rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3759        }
3760
3761        si_enable_gui_idle_interrupt(rdev, true);
3762
3763        if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3764                radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3765
3766        return 0;
3767}
3768
3769u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3770{
3771        u32 reset_mask = 0;
3772        u32 tmp;
3773
3774        /* GRBM_STATUS */
3775        tmp = RREG32(GRBM_STATUS);
3776        if (tmp & (PA_BUSY | SC_BUSY |
3777                   BCI_BUSY | SX_BUSY |
3778                   TA_BUSY | VGT_BUSY |
3779                   DB_BUSY | CB_BUSY |
3780                   GDS_BUSY | SPI_BUSY |
3781                   IA_BUSY | IA_BUSY_NO_DMA))
3782                reset_mask |= RADEON_RESET_GFX;
3783
3784        if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3785                   CP_BUSY | CP_COHERENCY_BUSY))
3786                reset_mask |= RADEON_RESET_CP;
3787
3788        if (tmp & GRBM_EE_BUSY)
3789                reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3790
3791        /* GRBM_STATUS2 */
3792        tmp = RREG32(GRBM_STATUS2);
3793        if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3794                reset_mask |= RADEON_RESET_RLC;
3795
3796        /* DMA_STATUS_REG 0 */
3797        tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3798        if (!(tmp & DMA_IDLE))
3799                reset_mask |= RADEON_RESET_DMA;
3800
3801        /* DMA_STATUS_REG 1 */
3802        tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3803        if (!(tmp & DMA_IDLE))
3804                reset_mask |= RADEON_RESET_DMA1;
3805
3806        /* SRBM_STATUS2 */
3807        tmp = RREG32(SRBM_STATUS2);
3808        if (tmp & DMA_BUSY)
3809                reset_mask |= RADEON_RESET_DMA;
3810
3811        if (tmp & DMA1_BUSY)
3812                reset_mask |= RADEON_RESET_DMA1;
3813
3814        /* SRBM_STATUS */
3815        tmp = RREG32(SRBM_STATUS);
3816
3817        if (tmp & IH_BUSY)
3818                reset_mask |= RADEON_RESET_IH;
3819
3820        if (tmp & SEM_BUSY)
3821                reset_mask |= RADEON_RESET_SEM;
3822
3823        if (tmp & GRBM_RQ_PENDING)
3824                reset_mask |= RADEON_RESET_GRBM;
3825
3826        if (tmp & VMC_BUSY)
3827                reset_mask |= RADEON_RESET_VMC;
3828
3829        if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3830                   MCC_BUSY | MCD_BUSY))
3831                reset_mask |= RADEON_RESET_MC;
3832
3833        if (evergreen_is_display_hung(rdev))
3834                reset_mask |= RADEON_RESET_DISPLAY;
3835
3836        /* VM_L2_STATUS */
3837        tmp = RREG32(VM_L2_STATUS);
3838        if (tmp & L2_BUSY)
3839                reset_mask |= RADEON_RESET_VMC;
3840
3841        /* Skip MC reset as it's mostly likely not hung, just busy */
3842        if (reset_mask & RADEON_RESET_MC) {
3843                DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3844                reset_mask &= ~RADEON_RESET_MC;
3845        }
3846
3847        return reset_mask;
3848}
3849
3850static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3851{
3852        struct evergreen_mc_save save;
3853        u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3854        u32 tmp;
3855
3856        if (reset_mask == 0)
3857                return;
3858
3859        dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3860
3861        evergreen_print_gpu_status_regs(rdev);
3862        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3863                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3864        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3865                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3866
3867        /* disable PG/CG */
3868        si_fini_pg(rdev);
3869        si_fini_cg(rdev);
3870
3871        /* stop the rlc */
3872        si_rlc_stop(rdev);
3873
3874        /* Disable CP parsing/prefetching */
3875        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3876
3877        if (reset_mask & RADEON_RESET_DMA) {
3878                /* dma0 */
3879                tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3880                tmp &= ~DMA_RB_ENABLE;
3881                WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3882        }
3883        if (reset_mask & RADEON_RESET_DMA1) {
3884                /* dma1 */
3885                tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3886                tmp &= ~DMA_RB_ENABLE;
3887                WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3888        }
3889
3890        udelay(50);
3891
3892        evergreen_mc_stop(rdev, &save);
3893        if (evergreen_mc_wait_for_idle(rdev)) {
3894                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3895        }
3896
3897        if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3898                grbm_soft_reset = SOFT_RESET_CB |
3899                        SOFT_RESET_DB |
3900                        SOFT_RESET_GDS |
3901                        SOFT_RESET_PA |
3902                        SOFT_RESET_SC |
3903                        SOFT_RESET_BCI |
3904                        SOFT_RESET_SPI |
3905                        SOFT_RESET_SX |
3906                        SOFT_RESET_TC |
3907                        SOFT_RESET_TA |
3908                        SOFT_RESET_VGT |
3909                        SOFT_RESET_IA;
3910        }
3911
3912        if (reset_mask & RADEON_RESET_CP) {
3913                grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3914
3915                srbm_soft_reset |= SOFT_RESET_GRBM;
3916        }
3917
3918        if (reset_mask & RADEON_RESET_DMA)
3919                srbm_soft_reset |= SOFT_RESET_DMA;
3920
3921        if (reset_mask & RADEON_RESET_DMA1)
3922                srbm_soft_reset |= SOFT_RESET_DMA1;
3923
3924        if (reset_mask & RADEON_RESET_DISPLAY)
3925                srbm_soft_reset |= SOFT_RESET_DC;
3926
3927        if (reset_mask & RADEON_RESET_RLC)
3928                grbm_soft_reset |= SOFT_RESET_RLC;
3929
3930        if (reset_mask & RADEON_RESET_SEM)
3931                srbm_soft_reset |= SOFT_RESET_SEM;
3932
3933        if (reset_mask & RADEON_RESET_IH)
3934                srbm_soft_reset |= SOFT_RESET_IH;
3935
3936        if (reset_mask & RADEON_RESET_GRBM)
3937                srbm_soft_reset |= SOFT_RESET_GRBM;
3938
3939        if (reset_mask & RADEON_RESET_VMC)
3940                srbm_soft_reset |= SOFT_RESET_VMC;
3941
3942        if (reset_mask & RADEON_RESET_MC)
3943                srbm_soft_reset |= SOFT_RESET_MC;
3944
3945        if (grbm_soft_reset) {
3946                tmp = RREG32(GRBM_SOFT_RESET);
3947                tmp |= grbm_soft_reset;
3948                dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3949                WREG32(GRBM_SOFT_RESET, tmp);
3950                tmp = RREG32(GRBM_SOFT_RESET);
3951
3952                udelay(50);
3953
3954                tmp &= ~grbm_soft_reset;
3955                WREG32(GRBM_SOFT_RESET, tmp);
3956                tmp = RREG32(GRBM_SOFT_RESET);
3957        }
3958
3959        if (srbm_soft_reset) {
3960                tmp = RREG32(SRBM_SOFT_RESET);
3961                tmp |= srbm_soft_reset;
3962                dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3963                WREG32(SRBM_SOFT_RESET, tmp);
3964                tmp = RREG32(SRBM_SOFT_RESET);
3965
3966                udelay(50);
3967
3968                tmp &= ~srbm_soft_reset;
3969                WREG32(SRBM_SOFT_RESET, tmp);
3970                tmp = RREG32(SRBM_SOFT_RESET);
3971        }
3972
3973        /* Wait a little for things to settle down */
3974        udelay(50);
3975
3976        evergreen_mc_resume(rdev, &save);
3977        udelay(50);
3978
3979        evergreen_print_gpu_status_regs(rdev);
3980}
3981
3982static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3983{
3984        u32 tmp, i;
3985
3986        tmp = RREG32(CG_SPLL_FUNC_CNTL);
3987        tmp |= SPLL_BYPASS_EN;
3988        WREG32(CG_SPLL_FUNC_CNTL, tmp);
3989
3990        tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3991        tmp |= SPLL_CTLREQ_CHG;
3992        WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3993
3994        for (i = 0; i < rdev->usec_timeout; i++) {
3995                if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3996                        break;
3997                udelay(1);
3998        }
3999
4000        tmp = RREG32(CG_SPLL_FUNC_CNTL_2);

4001        tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
4002        WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
4003
4004        tmp = RREG32(MPLL_CNTL_MODE);
4005        tmp &= ~MPLL_MCLK_SEL;
4006        WREG32(MPLL_CNTL_MODE, tmp);
4007}
4008
4009static void si_spll_powerdown(struct radeon_device *rdev)
4010{
4011        u32 tmp;
4012
4013        tmp = RREG32(SPLL_CNTL_MODE);
4014        tmp |= SPLL_SW_DIR_CONTROL;
4015        WREG32(SPLL_CNTL_MODE, tmp);
4016
4017        tmp = RREG32(CG_SPLL_FUNC_CNTL);
4018        tmp |= SPLL_RESET;
4019        WREG32(CG_SPLL_FUNC_CNTL, tmp);
4020
4021        tmp = RREG32(CG_SPLL_FUNC_CNTL);
4022        tmp |= SPLL_SLEEP;
4023        WREG32(CG_SPLL_FUNC_CNTL, tmp);
4024
4025        tmp = RREG32(SPLL_CNTL_MODE);
4026        tmp &= ~SPLL_SW_DIR_CONTROL;
4027        WREG32(SPLL_CNTL_MODE, tmp);
4028}
4029
4030static void si_gpu_pci_config_reset(struct radeon_device *rdev)
4031{
4032        struct evergreen_mc_save save;
4033        u32 tmp, i;
4034
4035        dev_info(rdev->dev, "GPU pci config reset\n");
4036
4037        /* disable dpm? */
4038
4039        /* disable cg/pg */
4040        si_fini_pg(rdev);
4041        si_fini_cg(rdev);
4042
4043        /* Disable CP parsing/prefetching */
4044        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4045        /* dma0 */
4046        tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4047        tmp &= ~DMA_RB_ENABLE;
4048        WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4049        /* dma1 */
4050        tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4051        tmp &= ~DMA_RB_ENABLE;
4052        WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4053        /* XXX other engines? */
4054
4055        /* halt the rlc, disable cp internal ints */
4056        si_rlc_stop(rdev);
4057
4058        udelay(50);
4059
4060        /* disable mem access */
4061        evergreen_mc_stop(rdev, &save);
4062        if (evergreen_mc_wait_for_idle(rdev)) {
4063                dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4064        }
4065
4066        /* set mclk/sclk to bypass */
4067        si_set_clk_bypass_mode(rdev);
4068        /* powerdown spll */
4069        si_spll_powerdown(rdev);
4070        /* disable BM */
4071        pci_clear_master(rdev->pdev);
4072        /* reset */
4073        radeon_pci_config_reset(rdev);
4074        /* wait for asic to come out of reset */
4075        for (i = 0; i < rdev->usec_timeout; i++) {
4076                if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4077                        break;
4078                udelay(1);
4079        }
4080}
4081
4082int si_asic_reset(struct radeon_device *rdev, bool hard)
4083{
4084        u32 reset_mask;
4085
4086        if (hard) {
4087                si_gpu_pci_config_reset(rdev);
4088                return 0;
4089        }
4090
4091        reset_mask = si_gpu_check_soft_reset(rdev);
4092
4093        if (reset_mask)
4094                r600_set_bios_scratch_engine_hung(rdev, true);
4095
4096        /* try soft reset */
4097        si_gpu_soft_reset(rdev, reset_mask);
4098
4099        reset_mask = si_gpu_check_soft_reset(rdev);
4100
4101        /* try pci config reset */
4102        if (reset_mask && radeon_hard_reset)
4103                si_gpu_pci_config_reset(rdev);
4104
4105        reset_mask = si_gpu_check_soft_reset(rdev);
4106
4107        if (!reset_mask)
4108                r600_set_bios_scratch_engine_hung(rdev, false);
4109
4110        return 0;
4111}
4112
4113/**
4114 * si_gfx_is_lockup - Check if the GFX engine is locked up
4115 *
4116 * @rdev: radeon_device pointer
4117 * @ring: radeon_ring structure holding ring information
4118 *
4119 * Check if the GFX engine is locked up.
4120 * Returns true if the engine appears to be locked up, false if not.
4121 */
4122bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4123{
4124        u32 reset_mask = si_gpu_check_soft_reset(rdev);
4125
4126        if (!(reset_mask & (RADEON_RESET_GFX |
4127                            RADEON_RESET_COMPUTE |
4128                            RADEON_RESET_CP))) {
4129                radeon_ring_lockup_update(rdev, ring);
4130                return false;
4131        }
4132        return radeon_ring_test_lockup(rdev, ring);
4133}
4134
4135/* MC */
4136static void si_mc_program(struct radeon_device *rdev)
4137{
4138        struct evergreen_mc_save save;
4139        u32 tmp;
4140        int i, j;
4141
4142        /* Initialize HDP */
4143        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4144                WREG32((0x2c14 + j), 0x00000000);
4145                WREG32((0x2c18 + j), 0x00000000);
4146                WREG32((0x2c1c + j), 0x00000000);
4147                WREG32((0x2c20 + j), 0x00000000);
4148                WREG32((0x2c24 + j), 0x00000000);
4149        }
4150        WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4151
4152        evergreen_mc_stop(rdev, &save);
4153        if (radeon_mc_wait_for_idle(rdev)) {
4154                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4155        }
4156        if (!ASIC_IS_NODCE(rdev))
4157                /* Lockout access through VGA aperture*/
4158                WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4159        /* Update configuration */
4160        WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4161               rdev->mc.vram_start >> 12);
4162        WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4163               rdev->mc.vram_end >> 12);
4164        WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4165               rdev->vram_scratch.gpu_addr >> 12);
4166        tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4167        tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4168        WREG32(MC_VM_FB_LOCATION, tmp);
4169        /* XXX double check these! */
4170        WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4171        WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4172        WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4173        WREG32(MC_VM_AGP_BASE, 0);
4174        WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4175        WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4176        if (radeon_mc_wait_for_idle(rdev)) {
4177                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4178        }
4179        evergreen_mc_resume(rdev, &save);
4180        if (!ASIC_IS_NODCE(rdev)) {
4181                /* we need to own VRAM, so turn off the VGA renderer here
4182                 * to stop it overwriting our objects */
4183                rv515_vga_render_disable(rdev);
4184        }
4185}
4186
4187void si_vram_gtt_location(struct radeon_device *rdev,
4188                          struct radeon_mc *mc)
4189{
4190        if (mc->mc_vram_size > 0xFFC0000000ULL) {
4191                /* leave room for at least 1024M GTT */
4192                dev_warn(rdev->dev, "limiting VRAM\n");
4193                mc->real_vram_size = 0xFFC0000000ULL;
4194                mc->mc_vram_size = 0xFFC0000000ULL;
4195        }
4196        radeon_vram_location(rdev, &rdev->mc, 0);
4197        rdev->mc.gtt_base_align = 0;
4198        radeon_gtt_location(rdev, mc);
4199}
4200
4201static int si_mc_init(struct radeon_device *rdev)
4202{
4203        u32 tmp;
4204        int chansize, numchan;
4205
4206        /* Get VRAM informations */
4207        rdev->mc.vram_is_ddr = true;
4208        tmp = RREG32(MC_ARB_RAMCFG);
4209        if (tmp & CHANSIZE_OVERRIDE) {
4210                chansize = 16;
4211        } else if (tmp & CHANSIZE_MASK) {
4212                chansize = 64;
4213        } else {
4214                chansize = 32;
4215        }
4216        tmp = RREG32(MC_SHARED_CHMAP);
4217        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4218        case 0:
4219        default:
4220                numchan = 1;
4221                break;
4222        case 1:
4223                numchan = 2;
4224                break;
4225        case 2:
4226                numchan = 4;
4227                break;
4228        case 3:
4229                numchan = 8;
4230                break;
4231        case 4:
4232                numchan = 3;
4233                break;
4234        case 5:
4235                numchan = 6;
4236                break;
4237        case 6:
4238                numchan = 10;
4239                break;
4240        case 7:
4241                numchan = 12;
4242                break;
4243        case 8:
4244                numchan = 16;
4245                break;
4246        }
4247        rdev->mc.vram_width = numchan * chansize;
4248        /* Could aper size report 0 ? */
4249        rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4250        rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4251        /* size in MB on si */
4252        tmp = RREG32(CONFIG_MEMSIZE);
4253        /* some boards may have garbage in the upper 16 bits */
4254        if (tmp & 0xffff0000) {
4255                DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4256                if (tmp & 0xffff)
4257                        tmp &= 0xffff;
4258        }
4259        rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4260        rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4261        rdev->mc.visible_vram_size = rdev->mc.aper_size;
4262        si_vram_gtt_location(rdev, &rdev->mc);
4263        radeon_update_bandwidth_info(rdev);
4264
4265        return 0;
4266}
4267
4268/*
4269 * GART
4270 */
4271void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4272{
4273        /* flush hdp cache */
4274        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4275
4276        /* bits 0-15 are the VM contexts0-15 */
4277        WREG32(VM_INVALIDATE_REQUEST, 1);
4278}
4279
4280static int si_pcie_gart_enable(struct radeon_device *rdev)
4281{
4282        int r, i;
4283
4284        if (rdev->gart.robj == NULL) {
4285                dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4286                return -EINVAL;
4287        }
4288        r = radeon_gart_table_vram_pin(rdev);
4289        if (r)
4290                return r;
4291        /* Setup TLB control */
4292        WREG32(MC_VM_MX_L1_TLB_CNTL,
4293               (0xA << 7) |
4294               ENABLE_L1_TLB |
4295               ENABLE_L1_FRAGMENT_PROCESSING |
4296               SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4297               ENABLE_ADVANCED_DRIVER_MODEL |
4298               SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4299        /* Setup L2 cache */
4300        WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4301               ENABLE_L2_FRAGMENT_PROCESSING |
4302               ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4303               ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4304               EFFECTIVE_L2_QUEUE_SIZE(7) |
4305               CONTEXT1_IDENTITY_ACCESS_MODE(1));
4306        WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4307        WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4308               BANK_SELECT(4) |
4309               L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4310        /* setup context0 */
4311        WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4312        WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4313        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4314        WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4315                        (u32)(rdev->dummy_page.addr >> 12));
4316        WREG32(VM_CONTEXT0_CNTL2, 0);
4317        WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4318                                  RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4319
4320        WREG32(0x15D4, 0);
4321        WREG32(0x15D8, 0);
4322        WREG32(0x15DC, 0);
4323
4324        /* empty context1-15 */
4325        /* set vm size, must be a multiple of 4 */
4326        WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4327        WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
4328        /* Assign the pt base to something valid for now; the pts used for
4329         * the VMs are determined by the application and setup and assigned
4330         * on the fly in the vm part of radeon_gart.c
4331         */
4332        for (i = 1; i < 16; i++) {
4333                if (i < 8)
4334                        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4335                               rdev->vm_manager.saved_table_addr[i]);
4336                else
4337                        WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4338                               rdev->vm_manager.saved_table_addr[i]);
4339        }
4340
4341        /* enable context1-15 */
4342        WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4343               (u32)(rdev->dummy_page.addr >> 12));
4344        WREG32(VM_CONTEXT1_CNTL2, 4);
4345        WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4346                                PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4347                                RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4348                                RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4349                                DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4350                                DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4351                                PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4352                                PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4353                                VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4354                                VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4355                                READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4356                                READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4357                                WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4358                                WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4359
4360        si_pcie_gart_tlb_flush(rdev);
4361        DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4362                 (unsigned)(rdev->mc.gtt_size >> 20),
4363                 (unsigned long long)rdev->gart.table_addr);
4364        rdev->gart.ready = true;
4365        return 0;
4366}
4367
4368static void si_pcie_gart_disable(struct radeon_device *rdev)
4369{
4370        unsigned i;
4371
4372        for (i = 1; i < 16; ++i) {
4373                uint32_t reg;
4374                if (i < 8)
4375                        reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4376                else
4377                        reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4378                rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4379        }
4380
4381        /* Disable all tables */
4382        WREG32(VM_CONTEXT0_CNTL, 0);
4383        WREG32(VM_CONTEXT1_CNTL, 0);
4384        /* Setup TLB control */
4385        WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4386               SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4387        /* Setup L2 cache */
4388        WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4389               ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4390               EFFECTIVE_L2_QUEUE_SIZE(7) |
4391               CONTEXT1_IDENTITY_ACCESS_MODE(1));
4392        WREG32(VM_L2_CNTL2, 0);
4393        WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4394               L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4395        radeon_gart_table_vram_unpin(rdev);
4396}
4397
4398static void si_pcie_gart_fini(struct radeon_device *rdev)
4399{
4400        si_pcie_gart_disable(rdev);
4401        radeon_gart_table_vram_free(rdev);
4402        radeon_gart_fini(rdev);
4403}
4404
4405/* vm parser */
4406static bool si_vm_reg_valid(u32 reg)
4407{
4408        /* context regs are fine */
4409        if (reg >= 0x28000)
4410                return true;
4411
4412        /* shader regs are also fine */
4413        if (reg >= 0xB000 && reg < 0xC000)
4414                return true;
4415
4416        /* check config regs */
4417        switch (reg) {
4418        case GRBM_GFX_INDEX:
4419        case CP_STRMOUT_CNTL:
4420        case VGT_VTX_VECT_EJECT_REG:
4421        case VGT_CACHE_INVALIDATION:
4422        case VGT_ESGS_RING_SIZE:
4423        case VGT_GSVS_RING_SIZE:
4424        case VGT_GS_VERTEX_REUSE:
4425        case VGT_PRIMITIVE_TYPE:
4426        case VGT_INDEX_TYPE:
4427        case VGT_NUM_INDICES:
4428        case VGT_NUM_INSTANCES:
4429        case VGT_TF_RING_SIZE:
4430        case VGT_HS_OFFCHIP_PARAM:
4431        case VGT_TF_MEMORY_BASE:
4432        case PA_CL_ENHANCE:
4433        case PA_SU_LINE_STIPPLE_VALUE:
4434        case PA_SC_LINE_STIPPLE_STATE:
4435        case PA_SC_ENHANCE:
4436        case SQC_CACHES:
4437        case SPI_STATIC_THREAD_MGMT_1:
4438        case SPI_STATIC_THREAD_MGMT_2:
4439        case SPI_STATIC_THREAD_MGMT_3:
4440        case SPI_PS_MAX_WAVE_ID:
4441        case SPI_CONFIG_CNTL:
4442        case SPI_CONFIG_CNTL_1:
4443        case TA_CNTL_AUX:
4444        case TA_CS_BC_BASE_ADDR:
4445                return true;
4446        default:
4447                DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4448                return false;
4449        }
4450}
4451
4452static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4453                                  u32 *ib, struct radeon_cs_packet *pkt)
4454{
4455        switch (pkt->opcode) {
4456        case PACKET3_NOP:
4457        case PACKET3_SET_BASE:
4458        case PACKET3_SET_CE_DE_COUNTERS:
4459        case PACKET3_LOAD_CONST_RAM:
4460        case PACKET3_WRITE_CONST_RAM:
4461        case PACKET3_WRITE_CONST_RAM_OFFSET:
4462        case PACKET3_DUMP_CONST_RAM:
4463        case PACKET3_INCREMENT_CE_COUNTER:
4464        case PACKET3_WAIT_ON_DE_COUNTER:
4465        case PACKET3_CE_WRITE:
4466                break;
4467        default:
4468                DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4469                return -EINVAL;
4470        }
4471        return 0;
4472}
4473
4474static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4475{
4476        u32 start_reg, reg, i;
4477        u32 command = ib[idx + 4];
4478        u32 info = ib[idx + 1];
4479        u32 idx_value = ib[idx];
4480        if (command & PACKET3_CP_DMA_CMD_SAS) {
4481                /* src address space is register */
4482                if (((info & 0x60000000) >> 29) == 0) {
4483                        start_reg = idx_value << 2;
4484                        if (command & PACKET3_CP_DMA_CMD_SAIC) {
4485                                reg = start_reg;
4486                                if (!si_vm_reg_valid(reg)) {
4487                                        DRM_ERROR("CP DMA Bad SRC register\n");
4488                                        return -EINVAL;
4489                                }
4490                        } else {
4491                                for (i = 0; i < (command & 0x1fffff); i++) {
4492                                        reg = start_reg + (4 * i);
4493                                        if (!si_vm_reg_valid(reg)) {
4494                                                DRM_ERROR("CP DMA Bad SRC register\n");
4495                                                return -EINVAL;
4496                                        }
4497                                }
4498                        }
4499                }
4500        }
4501        if (command & PACKET3_CP_DMA_CMD_DAS) {
4502                /* dst address space is register */
4503                if (((info & 0x00300000) >> 20) == 0) {
4504                        start_reg = ib[idx + 2];
4505                        if (command & PACKET3_CP_DMA_CMD_DAIC) {
4506                                reg = start_reg;
4507                                if (!si_vm_reg_valid(reg)) {
4508                                        DRM_ERROR("CP DMA Bad DST register\n");
4509                                        return -EINVAL;
4510                                }
4511                        } else {
4512                                for (i = 0; i < (command & 0x1fffff); i++) {
4513                                        reg = start_reg + (4 * i);
4514                                        if (!si_vm_reg_valid(reg)) {
4515                                                DRM_ERROR("CP DMA Bad DST register\n");
4516                                                return -EINVAL;
4517                                        }
4518                                }
4519                        }
4520                }
4521        }
4522        return 0;
4523}
4524
4525static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4526                                   u32 *ib, struct radeon_cs_packet *pkt)
4527{
4528        int r;
4529        u32 idx = pkt->idx + 1;
4530        u32 idx_value = ib[idx];
4531        u32 start_reg, end_reg, reg, i;
4532
4533        switch (pkt->opcode) {
4534        case PACKET3_NOP:
4535        case PACKET3_SET_BASE:
4536        case PACKET3_CLEAR_STATE:
4537        case PACKET3_INDEX_BUFFER_SIZE:
4538        case PACKET3_DISPATCH_DIRECT:
4539        case PACKET3_DISPATCH_INDIRECT:
4540        case PACKET3_ALLOC_GDS:
4541        case PACKET3_WRITE_GDS_RAM:
4542        case PACKET3_ATOMIC_GDS:
4543        case PACKET3_ATOMIC:
4544        case PACKET3_OCCLUSION_QUERY:
4545        case PACKET3_SET_PREDICATION:
4546        case PACKET3_COND_EXEC:
4547        case PACKET3_PRED_EXEC:
4548        case PACKET3_DRAW_INDIRECT:
4549        case PACKET3_DRAW_INDEX_INDIRECT:
4550        case PACKET3_INDEX_BASE:
4551        case PACKET3_DRAW_INDEX_2:
4552        case PACKET3_CONTEXT_CONTROL:
4553        case PACKET3_INDEX_TYPE:
4554        case PACKET3_DRAW_INDIRECT_MULTI:
4555        case PACKET3_DRAW_INDEX_AUTO:
4556        case PACKET3_DRAW_INDEX_IMMD:
4557        case PACKET3_NUM_INSTANCES:
4558        case PACKET3_DRAW_INDEX_MULTI_AUTO:
4559        case PACKET3_STRMOUT_BUFFER_UPDATE:
4560        case PACKET3_DRAW_INDEX_OFFSET_2:
4561        case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4562        case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4563        case PACKET3_MPEG_INDEX:
4564        case PACKET3_WAIT_REG_MEM:
4565        case PACKET3_MEM_WRITE:
4566        case PACKET3_PFP_SYNC_ME:
4567        case PACKET3_SURFACE_SYNC:
4568        case PACKET3_EVENT_WRITE:
4569        case PACKET3_EVENT_WRITE_EOP:
4570        case PACKET3_EVENT_WRITE_EOS:
4571        case PACKET3_SET_CONTEXT_REG:
4572        case PACKET3_SET_CONTEXT_REG_INDIRECT:
4573        case PACKET3_SET_SH_REG:
4574        case PACKET3_SET_SH_REG_OFFSET:
4575        case PACKET3_INCREMENT_DE_COUNTER:
4576        case PACKET3_WAIT_ON_CE_COUNTER:
4577        case PACKET3_WAIT_ON_AVAIL_BUFFER:
4578        case PACKET3_ME_WRITE:
4579                break;
4580        case PACKET3_COPY_DATA:
4581                if ((idx_value & 0xf00) == 0) {
4582                        reg = ib[idx + 3] * 4;
4583                        if (!si_vm_reg_valid(reg))
4584                                return -EINVAL;
4585                }
4586                break;
4587        case PACKET3_WRITE_DATA:
4588                if ((idx_value & 0xf00) == 0) {
4589                        start_reg = ib[idx + 1] * 4;
4590                        if (idx_value & 0x10000) {
4591                                if (!si_vm_reg_valid(start_reg))
4592                                        return -EINVAL;
4593                        } else {
4594                                for (i = 0; i < (pkt->count - 2); i++) {
4595                                        reg = start_reg + (4 * i);
4596                                        if (!si_vm_reg_valid(reg))
4597                                                return -EINVAL;
4598                                }
4599                        }
4600                }
4601                break;
4602        case PACKET3_COND_WRITE:
4603                if (idx_value & 0x100) {
4604                        reg = ib[idx + 5] * 4;
4605                        if (!si_vm_reg_valid(reg))
4606                                return -EINVAL;
4607                }
4608                break;
4609        case PACKET3_COPY_DW:
4610                if (idx_value & 0x2) {
4611                        reg = ib[idx + 3] * 4;
4612                        if (!si_vm_reg_valid(reg))
4613                                return -EINVAL;
4614                }
4615                break;
4616        case PACKET3_SET_CONFIG_REG:
4617                start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4618                end_reg = 4 * pkt->count + start_reg - 4;
4619                if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4620                    (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4621                    (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4622                        DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4623                        return -EINVAL;
4624                }
4625                for (i = 0; i < pkt->count; i++) {
4626                        reg = start_reg + (4 * i);
4627                        if (!si_vm_reg_valid(reg))
4628                                return -EINVAL;
4629                }
4630                break;
4631        case PACKET3_CP_DMA:
4632                r = si_vm_packet3_cp_dma_check(ib, idx);
4633                if (r)
4634                        return r;
4635                break;
4636        default:
4637                DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4638                return -EINVAL;
4639        }
4640        return 0;
4641}
4642
4643static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4644                                       u32 *ib, struct radeon_cs_packet *pkt)
4645{
4646        int r;
4647        u32 idx = pkt->idx + 1;
4648        u32 idx_value = ib[idx];
4649        u32 start_reg, reg, i;
4650
4651        switch (pkt->opcode) {
4652        case PACKET3_NOP:
4653        case PACKET3_SET_BASE:
4654        case PACKET3_CLEAR_STATE:
4655        case PACKET3_DISPATCH_DIRECT:
4656        case PACKET3_DISPATCH_INDIRECT:
4657        case PACKET3_ALLOC_GDS:
4658        case PACKET3_WRITE_GDS_RAM:
4659        case PACKET3_ATOMIC_GDS:
4660        case PACKET3_ATOMIC:
4661        case PACKET3_OCCLUSION_QUERY:
4662        case PACKET3_SET_PREDICATION:
4663        case PACKET3_COND_EXEC:
4664        case PACKET3_PRED_EXEC:
4665        case PACKET3_CONTEXT_CONTROL:
4666        case PACKET3_STRMOUT_BUFFER_UPDATE:
4667        case PACKET3_WAIT_REG_MEM:
4668        case PACKET3_MEM_WRITE:
4669        case PACKET3_PFP_SYNC_ME:
4670        case PACKET3_SURFACE_SYNC:
4671        case PACKET3_EVENT_WRITE:
4672        case PACKET3_EVENT_WRITE_EOP:
4673        case PACKET3_EVENT_WRITE_EOS:
4674        case PACKET3_SET_CONTEXT_REG:
4675        case PACKET3_SET_CONTEXT_REG_INDIRECT:
4676        case PACKET3_SET_SH_REG:
4677        case PACKET3_SET_SH_REG_OFFSET:
4678        case PACKET3_INCREMENT_DE_COUNTER:
4679        case PACKET3_WAIT_ON_CE_COUNTER:
4680        case PACKET3_WAIT_ON_AVAIL_BUFFER:
4681        case PACKET3_ME_WRITE:
4682                break;
4683        case PACKET3_COPY_DATA:
4684                if ((idx_value & 0xf00) == 0) {
4685                        reg = ib[idx + 3] * 4;
4686                        if (!si_vm_reg_valid(reg))
4687                                return -EINVAL;
4688                }
4689                break;
4690        case PACKET3_WRITE_DATA:
4691                if ((idx_value & 0xf00) == 0) {
4692                        start_reg = ib[idx + 1] * 4;
4693                        if (idx_value & 0x10000) {
4694                                if (!si_vm_reg_valid(start_reg))
4695                                        return -EINVAL;
4696                        } else {
4697                                for (i = 0; i < (pkt->count - 2); i++) {
4698                                        reg = start_reg + (4 * i);
4699                                        if (!si_vm_reg_valid(reg))
4700                                                return -EINVAL;
4701                                }
4702                        }
4703                }
4704                break;
4705        case PACKET3_COND_WRITE:
4706                if (idx_value & 0x100) {
4707                        reg = ib[idx + 5] * 4;
4708                        if (!si_vm_reg_valid(reg))
4709                                return -EINVAL;
4710                }
4711                break;
4712        case PACKET3_COPY_DW:
4713                if (idx_value & 0x2) {
4714                        reg = ib[idx + 3] * 4;
4715                        if (!si_vm_reg_valid(reg))
4716                                return -EINVAL;
4717                }
4718                break;
4719        case PACKET3_CP_DMA:
4720                r = si_vm_packet3_cp_dma_check(ib, idx);
4721                if (r)
4722                        return r;
4723                break;
4724        default:
4725                DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4726                return -EINVAL;
4727        }
4728        return 0;
4729}
4730
4731int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4732{
4733        int ret = 0;
4734        u32 idx = 0, i;
4735        struct radeon_cs_packet pkt;
4736
4737        do {
4738                pkt.idx = idx;
4739                pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4740                pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4741                pkt.one_reg_wr = 0;
4742                switch (pkt.type) {
4743                case RADEON_PACKET_TYPE0:
4744                        dev_err(rdev->dev, "Packet0 not allowed!\n");
4745                        ret = -EINVAL;
4746                        break;
4747                case RADEON_PACKET_TYPE2:
4748                        idx += 1;
4749                        break;
4750                case RADEON_PACKET_TYPE3:
4751                        pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4752                        if (ib->is_const_ib)
4753                                ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4754                        else {
4755                                switch (ib->ring) {
4756                                case RADEON_RING_TYPE_GFX_INDEX:
4757                                        ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4758                                        break;
4759                                case CAYMAN_RING_TYPE_CP1_INDEX:
4760                                case CAYMAN_RING_TYPE_CP2_INDEX:
4761                                        ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4762                                        break;
4763                                default:
4764                                        dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4765                                        ret = -EINVAL;
4766                                        break;
4767                                }
4768                        }
4769                        idx += pkt.count + 2;
4770                        break;
4771                default:
4772                        dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4773                        ret = -EINVAL;
4774                        break;
4775                }
4776                if (ret) {
4777                        for (i = 0; i < ib->length_dw; i++) {
4778                                if (i == idx)
4779                                        printk("\t0x%08x <---\n", ib->ptr[i]);
4780                                else
4781                                        printk("\t0x%08x\n", ib->ptr[i]);
4782                        }
4783                        break;
4784                }
4785        } while (idx < ib->length_dw);
4786
4787        return ret;
4788}
4789
4790/*
4791 * vm
4792 */
4793int si_vm_init(struct radeon_device *rdev)
4794{
4795        /* number of VMs */
4796        rdev->vm_manager.nvm = 16;
4797        /* base offset of vram pages */
4798        rdev->vm_manager.vram_base_offset = 0;
4799
4800        return 0;
4801}
4802
4803void si_vm_fini(struct radeon_device *rdev)
4804{
4805}
4806
4807/**
4808 * si_vm_decode_fault - print human readable fault info
4809 *
4810 * @rdev: radeon_device pointer
4811 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4812 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4813 *
4814 * Print human readable fault information (SI).
4815 */
4816static void si_vm_decode_fault(struct radeon_device *rdev,
4817                               u32 status, u32 addr)
4818{
4819        u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4820        u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4821        u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4822        char *block;
4823
4824        if (rdev->family == CHIP_TAHITI) {
4825                switch (mc_id) {
4826                case 160:
4827                case 144:
4828                case 96:
4829                case 80:
4830                case 224:
4831                case 208:
4832                case 32:
4833                case 16:
4834                        block = "CB";
4835                        break;
4836                case 161:
4837                case 145:
4838                case 97:
4839                case 81:
4840                case 225:
4841                case 209:
4842                case 33:
4843                case 17:
4844                        block = "CB_FMASK";
4845                        break;
4846                case 162:
4847                case 146:
4848                case 98:
4849                case 82:
4850                case 226:
4851                case 210:
4852                case 34:
4853                case 18:
4854                        block = "CB_CMASK";
4855                        break;
4856                case 163:
4857                case 147:
4858                case 99:
4859                case 83:
4860                case 227:
4861                case 211:
4862                case 35:
4863                case 19:
4864                        block = "CB_IMMED";
4865                        break;
4866                case 164:
4867                case 148:
4868                case 100:
4869                case 84:
4870                case 228:
4871                case 212:
4872                case 36:
4873                case 20:
4874                        block = "DB";
4875                        break;
4876                case 165:
4877                case 149:
4878                case 101:
4879                case 85:
4880                case 229:
4881                case 213:
4882                case 37:
4883                case 21:
4884                        block = "DB_HTILE";
4885                        break;
4886                case 167:
4887                case 151:
4888                case 103:
4889                case 87:
4890                case 231:
4891                case 215:
4892                case 39:
4893                case 23:
4894                        block = "DB_STEN";
4895                        break;
4896                case 72:
4897                case 68:
4898                case 64:
4899                case 8:
4900                case 4:
4901                case 0:
4902                case 136:
4903                case 132:
4904                case 128:
4905                case 200:
4906                case 196:
4907                case 192:
4908                        block = "TC";
4909                        break;
4910                case 112:
4911                case 48:
4912                        block = "CP";
4913                        break;
4914                case 49:
4915                case 177:
4916                case 50:
4917                case 178:
4918                        block = "SH";
4919                        break;
4920                case 53:
4921                case 190:
4922                        block = "VGT";
4923                        break;
4924                case 117:
4925                        block = "IH";
4926                        break;
4927                case 51:
4928                case 115:
4929                        block = "RLC";
4930                        break;
4931                case 119:
4932                case 183:
4933                        block = "DMA0";
4934                        break;
4935                case 61:
4936                        block = "DMA1";
4937                        break;
4938                case 248:
4939                case 120:
4940                        block = "HDP";
4941                        break;
4942                default:
4943                        block = "unknown";
4944                        break;
4945                }
4946        } else {
4947                switch (mc_id) {
4948                case 32:
4949                case 16:
4950                case 96:
4951                case 80:
4952                case 160:
4953                case 144:
4954                case 224:
4955                case 208:
4956                        block = "CB";
4957                        break;
4958                case 33:
4959                case 17:
4960                case 97:
4961                case 81:
4962                case 161:
4963                case 145:
4964                case 225:
4965                case 209:
4966                        block = "CB_FMASK";
4967                        break;
4968                case 34:
4969                case 18:
4970                case 98:
4971                case 82:
4972                case 162:
4973                case 146:
4974                case 226:
4975                case 210:
4976                        block = "CB_CMASK";
4977                        break;
4978                case 35:
4979                case 19:
4980                case 99:
4981                case 83:
4982                case 163:
4983                case 147:
4984                case 227:
4985                case 211:
4986                        block = "CB_IMMED";
4987                        break;
4988                case 36:
4989                case 20:
4990                case 100:
4991                case 84:
4992                case 164:
4993                case 148:
4994                case 228:
4995                case 212:
4996                        block = "DB";
4997                        break;
4998                case 37:
4999                case 21:
5000                case 101:

5001                case 85:
5002                case 165:
5003                case 149:
5004                case 229:
5005                case 213:
5006                        block = "DB_HTILE";
5007                        break;
5008                case 39:
5009                case 23:
5010                case 103:
5011                case 87:
5012                case 167:
5013                case 151:
5014                case 231:
5015                case 215:
5016                        block = "DB_STEN";
5017                        break;
5018                case 72:
5019                case 68:
5020                case 8:
5021                case 4:
5022                case 136:
5023                case 132:
5024                case 200:
5025                case 196:
5026                        block = "TC";
5027                        break;
5028                case 112:
5029                case 48:
5030                        block = "CP";
5031                        break;
5032                case 49:
5033                case 177:
5034                case 50:
5035                case 178:
5036                        block = "SH";
5037                        break;
5038                case 53:
5039                        block = "VGT";
5040                        break;
5041                case 117:
5042                        block = "IH";
5043                        break;
5044                case 51:
5045                case 115:
5046                        block = "RLC";
5047                        break;
5048                case 119:
5049                case 183:
5050                        block = "DMA0";
5051                        break;
5052                case 61:
5053                        block = "DMA1";
5054                        break;
5055                case 248:
5056                case 120:
5057                        block = "HDP";
5058                        break;
5059                default:
5060                        block = "unknown";
5061                        break;
5062                }
5063        }
5064
5065        printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5066               protections, vmid, addr,
5067               (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5068               block, mc_id);
5069}
5070
5071void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5072                 unsigned vm_id, uint64_t pd_addr)
5073{
5074        /* write new base address */
5075        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5076        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5077                                 WRITE_DATA_DST_SEL(0)));
5078
5079        if (vm_id < 8) {
5080                radeon_ring_write(ring,
5081                                  (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5082        } else {
5083                radeon_ring_write(ring,
5084                                  (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5085        }
5086        radeon_ring_write(ring, 0);
5087        radeon_ring_write(ring, pd_addr >> 12);
5088
5089        /* flush hdp cache */
5090        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5091        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5092                                 WRITE_DATA_DST_SEL(0)));
5093        radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5094        radeon_ring_write(ring, 0);
5095        radeon_ring_write(ring, 0x1);
5096
5097        /* bits 0-15 are the VM contexts0-15 */
5098        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5099        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5100                                 WRITE_DATA_DST_SEL(0)));
5101        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5102        radeon_ring_write(ring, 0);
5103        radeon_ring_write(ring, 1 << vm_id);
5104
5105        /* wait for the invalidate to complete */
5106        radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5107        radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
5108                                 WAIT_REG_MEM_ENGINE(0))); /* me */
5109        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5110        radeon_ring_write(ring, 0);
5111        radeon_ring_write(ring, 0); /* ref */
5112        radeon_ring_write(ring, 0); /* mask */
5113        radeon_ring_write(ring, 0x20); /* poll interval */
5114
5115        /* sync PFP to ME, otherwise we might get invalid PFP reads */
5116        radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5117        radeon_ring_write(ring, 0x0);
5118}
5119
5120/*
5121 *  Power and clock gating
5122 */
5123static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5124{
5125        int i;
5126
5127        for (i = 0; i < rdev->usec_timeout; i++) {
5128                if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5129                        break;
5130                udelay(1);
5131        }
5132
5133        for (i = 0; i < rdev->usec_timeout; i++) {
5134                if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5135                        break;
5136                udelay(1);
5137        }
5138}
5139
5140static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5141                                         bool enable)
5142{
5143        u32 tmp = RREG32(CP_INT_CNTL_RING0);
5144        u32 mask;
5145        int i;
5146
5147        if (enable)
5148                tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5149        else
5150                tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5151        WREG32(CP_INT_CNTL_RING0, tmp);
5152
5153        if (!enable) {
5154                /* read a gfx register */
5155                tmp = RREG32(DB_DEPTH_INFO);
5156
5157                mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5158                for (i = 0; i < rdev->usec_timeout; i++) {
5159                        if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5160                                break;
5161                        udelay(1);
5162                }
5163        }
5164}
5165
5166static void si_set_uvd_dcm(struct radeon_device *rdev,
5167                           bool sw_mode)
5168{
5169        u32 tmp, tmp2;
5170
5171        tmp = RREG32(UVD_CGC_CTRL);
5172        tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5173        tmp |= DCM | CG_DT(1) | CLK_OD(4);
5174
5175        if (sw_mode) {
5176                tmp &= ~0x7ffff800;
5177                tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5178        } else {
5179                tmp |= 0x7ffff800;
5180                tmp2 = 0;
5181        }
5182
5183        WREG32(UVD_CGC_CTRL, tmp);
5184        WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5185}
5186
5187void si_init_uvd_internal_cg(struct radeon_device *rdev)
5188{
5189        bool hw_mode = true;
5190
5191        if (hw_mode) {
5192                si_set_uvd_dcm(rdev, false);
5193        } else {
5194                u32 tmp = RREG32(UVD_CGC_CTRL);
5195                tmp &= ~DCM;
5196                WREG32(UVD_CGC_CTRL, tmp);
5197        }
5198}
5199
5200static u32 si_halt_rlc(struct radeon_device *rdev)
5201{
5202        u32 data, orig;
5203
5204        orig = data = RREG32(RLC_CNTL);
5205
5206        if (data & RLC_ENABLE) {
5207                data &= ~RLC_ENABLE;
5208                WREG32(RLC_CNTL, data);
5209
5210                si_wait_for_rlc_serdes(rdev);
5211        }
5212
5213        return orig;
5214}
5215
5216static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5217{
5218        u32 tmp;
5219
5220        tmp = RREG32(RLC_CNTL);
5221        if (tmp != rlc)
5222                WREG32(RLC_CNTL, rlc);
5223}
5224
5225static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5226{
5227        u32 data, orig;
5228
5229        orig = data = RREG32(DMA_PG);
5230        if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5231                data |= PG_CNTL_ENABLE;
5232        else
5233                data &= ~PG_CNTL_ENABLE;
5234        if (orig != data)
5235                WREG32(DMA_PG, data);
5236}
5237
5238static void si_init_dma_pg(struct radeon_device *rdev)
5239{
5240        u32 tmp;
5241
5242        WREG32(DMA_PGFSM_WRITE,  0x00002000);
5243        WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5244
5245        for (tmp = 0; tmp < 5; tmp++)
5246                WREG32(DMA_PGFSM_WRITE, 0);
5247}
5248
5249static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5250                               bool enable)
5251{
5252        u32 tmp;
5253
5254        if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5255                tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5256                WREG32(RLC_TTOP_D, tmp);
5257
5258                tmp = RREG32(RLC_PG_CNTL);
5259                tmp |= GFX_PG_ENABLE;
5260                WREG32(RLC_PG_CNTL, tmp);
5261
5262                tmp = RREG32(RLC_AUTO_PG_CTRL);
5263                tmp |= AUTO_PG_EN;
5264                WREG32(RLC_AUTO_PG_CTRL, tmp);
5265        } else {
5266                tmp = RREG32(RLC_AUTO_PG_CTRL);
5267                tmp &= ~AUTO_PG_EN;
5268                WREG32(RLC_AUTO_PG_CTRL, tmp);
5269
5270                tmp = RREG32(DB_RENDER_CONTROL);
5271        }
5272}
5273
5274static void si_init_gfx_cgpg(struct radeon_device *rdev)
5275{
5276        u32 tmp;
5277
5278        WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5279
5280        tmp = RREG32(RLC_PG_CNTL);
5281        tmp |= GFX_PG_SRC;
5282        WREG32(RLC_PG_CNTL, tmp);
5283
5284        WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5285
5286        tmp = RREG32(RLC_AUTO_PG_CTRL);
5287
5288        tmp &= ~GRBM_REG_SGIT_MASK;
5289        tmp |= GRBM_REG_SGIT(0x700);
5290        tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5291        WREG32(RLC_AUTO_PG_CTRL, tmp);
5292}
5293
5294static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5295{
5296        u32 mask = 0, tmp, tmp1;
5297        int i;
5298
5299        si_select_se_sh(rdev, se, sh);
5300        tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5301        tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5302        si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5303
5304        tmp &= 0xffff0000;
5305
5306        tmp |= tmp1;
5307        tmp >>= 16;
5308
5309        for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5310                mask <<= 1;
5311                mask |= 1;
5312        }
5313
5314        return (~tmp) & mask;
5315}
5316
5317static void si_init_ao_cu_mask(struct radeon_device *rdev)
5318{
5319        u32 i, j, k, active_cu_number = 0;
5320        u32 mask, counter, cu_bitmap;
5321        u32 tmp = 0;
5322
5323        for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5324                for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5325                        mask = 1;
5326                        cu_bitmap = 0;
5327                        counter  = 0;
5328                        for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5329                                if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5330                                        if (counter < 2)
5331                                                cu_bitmap |= mask;
5332                                        counter++;
5333                                }
5334                                mask <<= 1;
5335                        }
5336
5337                        active_cu_number += counter;
5338                        tmp |= (cu_bitmap << (i * 16 + j * 8));
5339                }
5340        }
5341
5342        WREG32(RLC_PG_AO_CU_MASK, tmp);
5343
5344        tmp = RREG32(RLC_MAX_PG_CU);
5345        tmp &= ~MAX_PU_CU_MASK;
5346        tmp |= MAX_PU_CU(active_cu_number);
5347        WREG32(RLC_MAX_PG_CU, tmp);
5348}
5349
5350static void si_enable_cgcg(struct radeon_device *rdev,
5351                           bool enable)
5352{
5353        u32 data, orig, tmp;
5354
5355        orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5356
5357        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5358                si_enable_gui_idle_interrupt(rdev, true);
5359
5360                WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5361
5362                tmp = si_halt_rlc(rdev);
5363
5364                WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5365                WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5366                WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5367
5368                si_wait_for_rlc_serdes(rdev);
5369
5370                si_update_rlc(rdev, tmp);
5371
5372                WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5373
5374                data |= CGCG_EN | CGLS_EN;
5375        } else {
5376                si_enable_gui_idle_interrupt(rdev, false);
5377
5378                RREG32(CB_CGTT_SCLK_CTRL);
5379                RREG32(CB_CGTT_SCLK_CTRL);
5380                RREG32(CB_CGTT_SCLK_CTRL);
5381                RREG32(CB_CGTT_SCLK_CTRL);
5382
5383                data &= ~(CGCG_EN | CGLS_EN);
5384        }
5385
5386        if (orig != data)
5387                WREG32(RLC_CGCG_CGLS_CTRL, data);
5388}
5389
5390static void si_enable_mgcg(struct radeon_device *rdev,
5391                           bool enable)
5392{
5393        u32 data, orig, tmp = 0;
5394
5395        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5396                orig = data = RREG32(CGTS_SM_CTRL_REG);
5397                data = 0x96940200;
5398                if (orig != data)
5399                        WREG32(CGTS_SM_CTRL_REG, data);
5400
5401                if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5402                        orig = data = RREG32(CP_MEM_SLP_CNTL);
5403                        data |= CP_MEM_LS_EN;
5404                        if (orig != data)
5405                                WREG32(CP_MEM_SLP_CNTL, data);
5406                }
5407
5408                orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5409                data &= 0xffffffc0;
5410                if (orig != data)
5411                        WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5412
5413                tmp = si_halt_rlc(rdev);
5414
5415                WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5416                WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5417                WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5418
5419                si_update_rlc(rdev, tmp);
5420        } else {
5421                orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5422                data |= 0x00000003;
5423                if (orig != data)
5424                        WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5425
5426                data = RREG32(CP_MEM_SLP_CNTL);
5427                if (data & CP_MEM_LS_EN) {
5428                        data &= ~CP_MEM_LS_EN;
5429                        WREG32(CP_MEM_SLP_CNTL, data);
5430                }
5431                orig = data = RREG32(CGTS_SM_CTRL_REG);
5432                data |= LS_OVERRIDE | OVERRIDE;
5433                if (orig != data)
5434                        WREG32(CGTS_SM_CTRL_REG, data);
5435
5436                tmp = si_halt_rlc(rdev);
5437
5438                WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5439                WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5440                WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5441
5442                si_update_rlc(rdev, tmp);
5443        }
5444}
5445
5446static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5447                               bool enable)
5448{
5449        u32 orig, data, tmp;
5450
5451        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5452                tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5453                tmp |= 0x3fff;
5454                WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5455
5456                orig = data = RREG32(UVD_CGC_CTRL);
5457                data |= DCM;
5458                if (orig != data)
5459                        WREG32(UVD_CGC_CTRL, data);
5460
5461                WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5462                WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5463        } else {
5464                tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5465                tmp &= ~0x3fff;
5466                WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5467
5468                orig = data = RREG32(UVD_CGC_CTRL);
5469                data &= ~DCM;
5470                if (orig != data)
5471                        WREG32(UVD_CGC_CTRL, data);
5472
5473                WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5474                WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5475        }
5476}
5477
5478static const u32 mc_cg_registers[] =
5479{
5480        MC_HUB_MISC_HUB_CG,
5481        MC_HUB_MISC_SIP_CG,
5482        MC_HUB_MISC_VM_CG,
5483        MC_XPB_CLK_GAT,
5484        ATC_MISC_CG,
5485        MC_CITF_MISC_WR_CG,
5486        MC_CITF_MISC_RD_CG,
5487        MC_CITF_MISC_VM_CG,
5488        VM_L2_CG,
5489};
5490
5491static void si_enable_mc_ls(struct radeon_device *rdev,
5492                            bool enable)
5493{
5494        int i;
5495        u32 orig, data;
5496
5497        for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5498                orig = data = RREG32(mc_cg_registers[i]);
5499                if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5500                        data |= MC_LS_ENABLE;
5501                else
5502                        data &= ~MC_LS_ENABLE;
5503                if (data != orig)
5504                        WREG32(mc_cg_registers[i], data);
5505        }
5506}
5507
5508static void si_enable_mc_mgcg(struct radeon_device *rdev,
5509                               bool enable)
5510{
5511        int i;
5512        u32 orig, data;
5513
5514        for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5515                orig = data = RREG32(mc_cg_registers[i]);
5516                if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5517                        data |= MC_CG_ENABLE;
5518                else
5519                        data &= ~MC_CG_ENABLE;
5520                if (data != orig)
5521                        WREG32(mc_cg_registers[i], data);
5522        }
5523}
5524
5525static void si_enable_dma_mgcg(struct radeon_device *rdev,
5526                               bool enable)
5527{
5528        u32 orig, data, offset;
5529        int i;
5530
5531        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5532                for (i = 0; i < 2; i++) {
5533                        if (i == 0)
5534                                offset = DMA0_REGISTER_OFFSET;
5535                        else
5536                                offset = DMA1_REGISTER_OFFSET;
5537                        orig = data = RREG32(DMA_POWER_CNTL + offset);
5538                        data &= ~MEM_POWER_OVERRIDE;
5539                        if (data != orig)
5540                                WREG32(DMA_POWER_CNTL + offset, data);
5541                        WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5542                }
5543        } else {
5544                for (i = 0; i < 2; i++) {
5545                        if (i == 0)
5546                                offset = DMA0_REGISTER_OFFSET;
5547                        else
5548                                offset = DMA1_REGISTER_OFFSET;
5549                        orig = data = RREG32(DMA_POWER_CNTL + offset);
5550                        data |= MEM_POWER_OVERRIDE;
5551                        if (data != orig)
5552                                WREG32(DMA_POWER_CNTL + offset, data);
5553
5554                        orig = data = RREG32(DMA_CLK_CTRL + offset);
5555                        data = 0xff000000;
5556                        if (data != orig)
5557                                WREG32(DMA_CLK_CTRL + offset, data);
5558                }
5559        }
5560}
5561
5562static void si_enable_bif_mgls(struct radeon_device *rdev,
5563                               bool enable)
5564{
5565        u32 orig, data;
5566
5567        orig = data = RREG32_PCIE(PCIE_CNTL2);
5568
5569        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5570                data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5571                        REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5572        else
5573                data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5574                          REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5575
5576        if (orig != data)
5577                WREG32_PCIE(PCIE_CNTL2, data);
5578}
5579
5580static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5581                               bool enable)
5582{
5583        u32 orig, data;
5584
5585        orig = data = RREG32(HDP_HOST_PATH_CNTL);
5586
5587        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5588                data &= ~CLOCK_GATING_DIS;
5589        else
5590                data |= CLOCK_GATING_DIS;
5591
5592        if (orig != data)
5593                WREG32(HDP_HOST_PATH_CNTL, data);
5594}
5595
5596static void si_enable_hdp_ls(struct radeon_device *rdev,
5597                             bool enable)
5598{
5599        u32 orig, data;
5600
5601        orig = data = RREG32(HDP_MEM_POWER_LS);
5602
5603        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5604                data |= HDP_LS_ENABLE;
5605        else
5606                data &= ~HDP_LS_ENABLE;
5607
5608        if (orig != data)
5609                WREG32(HDP_MEM_POWER_LS, data);
5610}
5611
5612static void si_update_cg(struct radeon_device *rdev,
5613                         u32 block, bool enable)
5614{
5615        if (block & RADEON_CG_BLOCK_GFX) {
5616                si_enable_gui_idle_interrupt(rdev, false);
5617                /* order matters! */
5618                if (enable) {
5619                        si_enable_mgcg(rdev, true);
5620                        si_enable_cgcg(rdev, true);
5621                } else {
5622                        si_enable_cgcg(rdev, false);
5623                        si_enable_mgcg(rdev, false);
5624                }
5625                si_enable_gui_idle_interrupt(rdev, true);
5626        }
5627
5628        if (block & RADEON_CG_BLOCK_MC) {
5629                si_enable_mc_mgcg(rdev, enable);
5630                si_enable_mc_ls(rdev, enable);
5631        }
5632
5633        if (block & RADEON_CG_BLOCK_SDMA) {
5634                si_enable_dma_mgcg(rdev, enable);
5635        }
5636
5637        if (block & RADEON_CG_BLOCK_BIF) {
5638                si_enable_bif_mgls(rdev, enable);
5639        }
5640
5641        if (block & RADEON_CG_BLOCK_UVD) {
5642                if (rdev->has_uvd) {
5643                        si_enable_uvd_mgcg(rdev, enable);
5644                }
5645        }
5646
5647        if (block & RADEON_CG_BLOCK_HDP) {
5648                si_enable_hdp_mgcg(rdev, enable);
5649                si_enable_hdp_ls(rdev, enable);
5650        }
5651}
5652
5653static void si_init_cg(struct radeon_device *rdev)
5654{
5655        si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5656                            RADEON_CG_BLOCK_MC |
5657                            RADEON_CG_BLOCK_SDMA |
5658                            RADEON_CG_BLOCK_BIF |
5659                            RADEON_CG_BLOCK_HDP), true);
5660        if (rdev->has_uvd) {
5661                si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5662                si_init_uvd_internal_cg(rdev);
5663        }
5664}
5665
5666static void si_fini_cg(struct radeon_device *rdev)
5667{
5668        if (rdev->has_uvd) {
5669                si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5670        }
5671        si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5672                            RADEON_CG_BLOCK_MC |
5673                            RADEON_CG_BLOCK_SDMA |
5674                            RADEON_CG_BLOCK_BIF |
5675                            RADEON_CG_BLOCK_HDP), false);
5676}
5677
5678u32 si_get_csb_size(struct radeon_device *rdev)
5679{
5680        u32 count = 0;
5681        const struct cs_section_def *sect = NULL;
5682        const struct cs_extent_def *ext = NULL;
5683
5684        if (rdev->rlc.cs_data == NULL)
5685                return 0;
5686
5687        /* begin clear state */
5688        count += 2;
5689        /* context control state */
5690        count += 3;
5691
5692        for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5693                for (ext = sect->section; ext->extent != NULL; ++ext) {
5694                        if (sect->id == SECT_CONTEXT)
5695                                count += 2 + ext->reg_count;
5696                        else
5697                                return 0;
5698                }
5699        }
5700        /* pa_sc_raster_config */
5701        count += 3;
5702        /* end clear state */
5703        count += 2;
5704        /* clear state */
5705        count += 2;
5706
5707        return count;
5708}
5709
5710void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5711{
5712        u32 count = 0, i;
5713        const struct cs_section_def *sect = NULL;
5714        const struct cs_extent_def *ext = NULL;
5715
5716        if (rdev->rlc.cs_data == NULL)
5717                return;
5718        if (buffer == NULL)
5719                return;
5720
5721        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5722        buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5723
5724        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5725        buffer[count++] = cpu_to_le32(0x80000000);
5726        buffer[count++] = cpu_to_le32(0x80000000);
5727
5728        for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5729                for (ext = sect->section; ext->extent != NULL; ++ext) {
5730                        if (sect->id == SECT_CONTEXT) {
5731                                buffer[count++] =
5732                                        cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5733                                buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5734                                for (i = 0; i < ext->reg_count; i++)
5735                                        buffer[count++] = cpu_to_le32(ext->extent[i]);
5736                        } else {
5737                                return;
5738                        }
5739                }
5740        }
5741
5742        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5743        buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5744        switch (rdev->family) {
5745        case CHIP_TAHITI:
5746        case CHIP_PITCAIRN:
5747                buffer[count++] = cpu_to_le32(0x2a00126a);
5748                break;
5749        case CHIP_VERDE:
5750                buffer[count++] = cpu_to_le32(0x0000124a);
5751                break;
5752        case CHIP_OLAND:
5753                buffer[count++] = cpu_to_le32(0x00000082);
5754                break;
5755        case CHIP_HAINAN:
5756                buffer[count++] = cpu_to_le32(0x00000000);
5757                break;
5758        default:
5759                buffer[count++] = cpu_to_le32(0x00000000);
5760                break;
5761        }
5762
5763        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5764        buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5765
5766        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5767        buffer[count++] = cpu_to_le32(0);
5768}
5769
5770static void si_init_pg(struct radeon_device *rdev)
5771{
5772        if (rdev->pg_flags) {
5773                if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5774                        si_init_dma_pg(rdev);
5775                }
5776                si_init_ao_cu_mask(rdev);
5777                if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5778                        si_init_gfx_cgpg(rdev);
5779                } else {
5780                        WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5781                        WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5782                }
5783                si_enable_dma_pg(rdev, true);
5784                si_enable_gfx_cgpg(rdev, true);
5785        } else {
5786                WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5787                WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5788        }
5789}
5790
5791static void si_fini_pg(struct radeon_device *rdev)
5792{
5793        if (rdev->pg_flags) {
5794                si_enable_dma_pg(rdev, false);
5795                si_enable_gfx_cgpg(rdev, false);
5796        }
5797}
5798
5799/*
5800 * RLC
5801 */
5802void si_rlc_reset(struct radeon_device *rdev)
5803{
5804        u32 tmp = RREG32(GRBM_SOFT_RESET);
5805
5806        tmp |= SOFT_RESET_RLC;
5807        WREG32(GRBM_SOFT_RESET, tmp);
5808        udelay(50);
5809        tmp &= ~SOFT_RESET_RLC;
5810        WREG32(GRBM_SOFT_RESET, tmp);
5811        udelay(50);
5812}
5813
5814static void si_rlc_stop(struct radeon_device *rdev)
5815{
5816        WREG32(RLC_CNTL, 0);
5817
5818        si_enable_gui_idle_interrupt(rdev, false);
5819
5820        si_wait_for_rlc_serdes(rdev);
5821}
5822
5823static void si_rlc_start(struct radeon_device *rdev)
5824{
5825        WREG32(RLC_CNTL, RLC_ENABLE);
5826
5827        si_enable_gui_idle_interrupt(rdev, true);
5828
5829        udelay(50);
5830}
5831
5832static bool si_lbpw_supported(struct radeon_device *rdev)
5833{
5834        u32 tmp;
5835
5836        /* Enable LBPW only for DDR3 */
5837        tmp = RREG32(MC_SEQ_MISC0);
5838        if ((tmp & 0xF0000000) == 0xB0000000)
5839                return true;
5840        return false;
5841}
5842
5843static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5844{
5845        u32 tmp;
5846
5847        tmp = RREG32(RLC_LB_CNTL);
5848        if (enable)
5849                tmp |= LOAD_BALANCE_ENABLE;
5850        else
5851                tmp &= ~LOAD_BALANCE_ENABLE;
5852        WREG32(RLC_LB_CNTL, tmp);
5853
5854        if (!enable) {
5855                si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5856                WREG32(SPI_LB_CU_MASK, 0x00ff);
5857        }
5858}
5859
5860static int si_rlc_resume(struct radeon_device *rdev)
5861{
5862        u32 i;
5863
5864        if (!rdev->rlc_fw)
5865                return -EINVAL;
5866
5867        si_rlc_stop(rdev);
5868
5869        si_rlc_reset(rdev);
5870
5871        si_init_pg(rdev);
5872
5873        si_init_cg(rdev);
5874
5875        WREG32(RLC_RL_BASE, 0);
5876        WREG32(RLC_RL_SIZE, 0);
5877        WREG32(RLC_LB_CNTL, 0);
5878        WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5879        WREG32(RLC_LB_CNTR_INIT, 0);
5880        WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5881
5882        WREG32(RLC_MC_CNTL, 0);
5883        WREG32(RLC_UCODE_CNTL, 0);
5884
5885        if (rdev->new_fw) {
5886                const struct rlc_firmware_header_v1_0 *hdr =
5887                        (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5888                u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5889                const __le32 *fw_data = (const __le32 *)
5890                        (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5891
5892                radeon_ucode_print_rlc_hdr(&hdr->header);
5893
5894                for (i = 0; i < fw_size; i++) {
5895                        WREG32(RLC_UCODE_ADDR, i);
5896                        WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5897                }
5898        } else {
5899                const __be32 *fw_data =
5900                        (const __be32 *)rdev->rlc_fw->data;
5901                for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5902                        WREG32(RLC_UCODE_ADDR, i);
5903                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5904                }
5905        }
5906        WREG32(RLC_UCODE_ADDR, 0);
5907
5908        si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5909
5910        si_rlc_start(rdev);
5911
5912        return 0;
5913}
5914
5915static void si_enable_interrupts(struct radeon_device *rdev)
5916{
5917        u32 ih_cntl = RREG32(IH_CNTL);
5918        u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5919
5920        ih_cntl |= ENABLE_INTR;
5921        ih_rb_cntl |= IH_RB_ENABLE;
5922        WREG32(IH_CNTL, ih_cntl);
5923        WREG32(IH_RB_CNTL, ih_rb_cntl);
5924        rdev->ih.enabled = true;
5925}
5926
5927static void si_disable_interrupts(struct radeon_device *rdev)
5928{
5929        u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5930        u32 ih_cntl = RREG32(IH_CNTL);
5931
5932        ih_rb_cntl &= ~IH_RB_ENABLE;
5933        ih_cntl &= ~ENABLE_INTR;
5934        WREG32(IH_RB_CNTL, ih_rb_cntl);
5935        WREG32(IH_CNTL, ih_cntl);
5936        /* set rptr, wptr to 0 */
5937        WREG32(IH_RB_RPTR, 0);
5938        WREG32(IH_RB_WPTR, 0);
5939        rdev->ih.enabled = false;
5940        rdev->ih.rptr = 0;
5941}
5942
5943static void si_disable_interrupt_state(struct radeon_device *rdev)
5944{
5945        int i;
5946        u32 tmp;
5947
5948        tmp = RREG32(CP_INT_CNTL_RING0) &
5949                (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5950        WREG32(CP_INT_CNTL_RING0, tmp);
5951        WREG32(CP_INT_CNTL_RING1, 0);
5952        WREG32(CP_INT_CNTL_RING2, 0);
5953        tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5954        WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5955        tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5956        WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5957        WREG32(GRBM_INT_CNTL, 0);
5958        WREG32(SRBM_INT_CNTL, 0);
5959        for (i = 0; i < rdev->num_crtc; i++)
5960                WREG32(INT_MASK + crtc_offsets[i], 0);
5961        for (i = 0; i < rdev->num_crtc; i++)
5962                WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
5963
5964        if (!ASIC_IS_NODCE(rdev)) {
5965                WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5966
5967                for (i = 0; i < 6; i++)
5968                        WREG32_AND(DC_HPDx_INT_CONTROL(i),
5969                                   DC_HPDx_INT_POLARITY);
5970        }
5971}
5972
5973static int si_irq_init(struct radeon_device *rdev)
5974{
5975        int ret = 0;
5976        int rb_bufsz;
5977        u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5978
5979        /* allocate ring */
5980        ret = r600_ih_ring_alloc(rdev);
5981        if (ret)
5982                return ret;
5983
5984        /* disable irqs */
5985        si_disable_interrupts(rdev);
5986
5987        /* init rlc */
5988        ret = si_rlc_resume(rdev);
5989        if (ret) {
5990                r600_ih_ring_fini(rdev);
5991                return ret;
5992        }
5993
5994        /* setup interrupt control */
5995        /* set dummy read address to dummy page address */
5996        WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);
5997        interrupt_cntl = RREG32(INTERRUPT_CNTL);
5998        /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5999         * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6000         */

6001        interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6002        /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6003        interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6004        WREG32(INTERRUPT_CNTL, interrupt_cntl);
6005
6006        WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6007        rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6008
6009        ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6010                      IH_WPTR_OVERFLOW_CLEAR |
6011                      (rb_bufsz << 1));
6012
6013        if (rdev->wb.enabled)
6014                ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6015
6016        /* set the writeback address whether it's enabled or not */
6017        WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6018        WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6019
6020        WREG32(IH_RB_CNTL, ih_rb_cntl);
6021
6022        /* set rptr, wptr to 0 */
6023        WREG32(IH_RB_RPTR, 0);
6024        WREG32(IH_RB_WPTR, 0);
6025
6026        /* Default settings for IH_CNTL (disabled at first) */
6027        ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6028        /* RPTR_REARM only works if msi's are enabled */
6029        if (rdev->msi_enabled)
6030                ih_cntl |= RPTR_REARM;
6031        WREG32(IH_CNTL, ih_cntl);
6032
6033        /* force the active interrupt state to all disabled */
6034        si_disable_interrupt_state(rdev);
6035
6036        pci_set_master(rdev->pdev);
6037
6038        /* enable irqs */
6039        si_enable_interrupts(rdev);
6040
6041        return ret;
6042}
6043
6044/* The order we write back each register here is important */
6045int si_irq_set(struct radeon_device *rdev)
6046{
6047        int i;
6048        u32 cp_int_cntl;
6049        u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
6050        u32 grbm_int_cntl = 0;
6051        u32 dma_cntl, dma_cntl1;
6052        u32 thermal_int = 0;
6053
6054        if (!rdev->irq.installed) {
6055                WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6056                return -EINVAL;
6057        }
6058        /* don't enable anything if the ih is disabled */
6059        if (!rdev->ih.enabled) {
6060                si_disable_interrupts(rdev);
6061                /* force the active interrupt state to all disabled */
6062                si_disable_interrupt_state(rdev);
6063                return 0;
6064        }
6065
6066        cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6067                (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6068
6069        dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6070        dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6071
6072        thermal_int = RREG32(CG_THERMAL_INT) &
6073                ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6074
6075        /* enable CP interrupts on all rings */
6076        if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6077                DRM_DEBUG("si_irq_set: sw int gfx\n");
6078                cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6079        }
6080        if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6081                DRM_DEBUG("si_irq_set: sw int cp1\n");
6082                cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6083        }
6084        if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6085                DRM_DEBUG("si_irq_set: sw int cp2\n");
6086                cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6087        }
6088        if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6089                DRM_DEBUG("si_irq_set: sw int dma\n");
6090                dma_cntl |= TRAP_ENABLE;
6091        }
6092
6093        if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6094                DRM_DEBUG("si_irq_set: sw int dma1\n");
6095                dma_cntl1 |= TRAP_ENABLE;
6096        }
6097
6098        WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6099        WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6100        WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6101
6102        WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6103        WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6104
6105        WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6106
6107        if (rdev->irq.dpm_thermal) {
6108                DRM_DEBUG("dpm thermal\n");
6109                thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6110        }
6111
6112        for (i = 0; i < rdev->num_crtc; i++) {
6113                radeon_irq_kms_set_irq_n_enabled(
6114                    rdev, INT_MASK + crtc_offsets[i], VBLANK_INT_MASK,
6115                    rdev->irq.crtc_vblank_int[i] ||
6116                    atomic_read(&rdev->irq.pflip[i]), "vblank", i);
6117        }
6118
6119        for (i = 0; i < rdev->num_crtc; i++)
6120                WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
6121
6122        if (!ASIC_IS_NODCE(rdev)) {
6123                for (i = 0; i < 6; i++) {
6124                        radeon_irq_kms_set_irq_n_enabled(
6125                            rdev, DC_HPDx_INT_CONTROL(i),
6126                            DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
6127                            rdev->irq.hpd[i], "HPD", i);
6128                }
6129        }
6130
6131        WREG32(CG_THERMAL_INT, thermal_int);
6132
6133        /* posting read */
6134        RREG32(SRBM_STATUS);
6135
6136        return 0;
6137}
6138
6139/* The order we write back each register here is important */
6140static inline void si_irq_ack(struct radeon_device *rdev)
6141{
6142        int i, j;
6143        u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
6144        u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
6145
6146        if (ASIC_IS_NODCE(rdev))
6147                return;
6148
6149        for (i = 0; i < 6; i++) {
6150                disp_int[i] = RREG32(si_disp_int_status[i]);
6151                if (i < rdev->num_crtc)
6152                        grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
6153        }
6154
6155        /* We write back each interrupt register in pairs of two */
6156        for (i = 0; i < rdev->num_crtc; i += 2) {
6157                for (j = i; j < (i + 2); j++) {
6158                        if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
6159                                WREG32(GRPH_INT_STATUS + crtc_offsets[j],
6160                                       GRPH_PFLIP_INT_CLEAR);
6161                }
6162
6163                for (j = i; j < (i + 2); j++) {
6164                        if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
6165                                WREG32(VBLANK_STATUS + crtc_offsets[j],
6166                                       VBLANK_ACK);
6167                        if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
6168                                WREG32(VLINE_STATUS + crtc_offsets[j],
6169                                       VLINE_ACK);
6170                }
6171        }
6172
6173        for (i = 0; i < 6; i++) {
6174                if (disp_int[i] & DC_HPD1_INTERRUPT)
6175                        WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
6176        }
6177
6178        for (i = 0; i < 6; i++) {
6179                if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
6180                        WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
6181        }
6182}
6183
6184static void si_irq_disable(struct radeon_device *rdev)
6185{
6186        si_disable_interrupts(rdev);
6187        /* Wait and acknowledge irq */
6188        mdelay(1);
6189        si_irq_ack(rdev);
6190        si_disable_interrupt_state(rdev);
6191}
6192
6193static void si_irq_suspend(struct radeon_device *rdev)
6194{
6195        si_irq_disable(rdev);
6196        si_rlc_stop(rdev);
6197}
6198
6199static void si_irq_fini(struct radeon_device *rdev)
6200{
6201        si_irq_suspend(rdev);
6202        r600_ih_ring_fini(rdev);
6203}
6204
6205static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6206{
6207        u32 wptr, tmp;
6208
6209        if (rdev->wb.enabled)
6210                wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6211        else
6212                wptr = RREG32(IH_RB_WPTR);
6213
6214        if (wptr & RB_OVERFLOW) {
6215                wptr &= ~RB_OVERFLOW;
6216                /* When a ring buffer overflow happen start parsing interrupt
6217                 * from the last not overwritten vector (wptr + 16). Hopefully
6218                 * this should allow us to catchup.
6219                 */
6220                dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6221                         wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6222                rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6223                tmp = RREG32(IH_RB_CNTL);
6224                tmp |= IH_WPTR_OVERFLOW_CLEAR;
6225                WREG32(IH_RB_CNTL, tmp);
6226        }
6227        return (wptr & rdev->ih.ptr_mask);
6228}
6229
6230/*        SI IV Ring
6231 * Each IV ring entry is 128 bits:
6232 * [7:0]    - interrupt source id
6233 * [31:8]   - reserved
6234 * [59:32]  - interrupt source data
6235 * [63:60]  - reserved
6236 * [71:64]  - RINGID
6237 * [79:72]  - VMID
6238 * [127:80] - reserved
6239 */
6240int si_irq_process(struct radeon_device *rdev)
6241{
6242        u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
6243        u32 crtc_idx, hpd_idx;
6244        u32 mask;
6245        u32 wptr;
6246        u32 rptr;
6247        u32 src_id, src_data, ring_id;
6248        u32 ring_index;
6249        bool queue_hotplug = false;
6250        bool queue_dp = false;
6251        bool queue_thermal = false;
6252        u32 status, addr;
6253        const char *event_name;
6254
6255        if (!rdev->ih.enabled || rdev->shutdown)
6256                return IRQ_NONE;
6257
6258        wptr = si_get_ih_wptr(rdev);
6259
6260restart_ih:
6261        /* is somebody else already processing irqs? */
6262        if (atomic_xchg(&rdev->ih.lock, 1))
6263                return IRQ_NONE;
6264
6265        rptr = rdev->ih.rptr;
6266        DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6267
6268        /* Order reading of wptr vs. reading of IH ring data */
6269        rmb();
6270
6271        /* display interrupts */
6272        si_irq_ack(rdev);
6273
6274        while (rptr != wptr) {
6275                /* wptr/rptr are in bytes! */
6276                ring_index = rptr / 4;
6277                src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6278                src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6279                ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6280
6281                switch (src_id) {
6282                case 1: /* D1 vblank/vline */
6283                case 2: /* D2 vblank/vline */
6284                case 3: /* D3 vblank/vline */
6285                case 4: /* D4 vblank/vline */
6286                case 5: /* D5 vblank/vline */
6287                case 6: /* D6 vblank/vline */
6288                        crtc_idx = src_id - 1;
6289
6290                        if (src_data == 0) { /* vblank */
6291                                mask = LB_D1_VBLANK_INTERRUPT;
6292                                event_name = "vblank";
6293
6294                                if (rdev->irq.crtc_vblank_int[crtc_idx]) {
6295                                        drm_handle_vblank(rdev->ddev, crtc_idx);
6296                                        rdev->pm.vblank_sync = true;
6297                                        wake_up(&rdev->irq.vblank_queue);
6298                                }
6299                                if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
6300                                        radeon_crtc_handle_vblank(rdev,
6301                                                                  crtc_idx);
6302                                }
6303
6304                        } else if (src_data == 1) { /* vline */
6305                                mask = LB_D1_VLINE_INTERRUPT;
6306                                event_name = "vline";
6307                        } else {
6308                                DRM_DEBUG("Unhandled interrupt: %d %d\n",
6309                                          src_id, src_data);
6310                                break;
6311                        }
6312
6313                        if (!(disp_int[crtc_idx] & mask)) {
6314                                DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
6315                                          crtc_idx + 1, event_name);
6316                        }
6317
6318                        disp_int[crtc_idx] &= ~mask;
6319                        DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
6320
6321                        break;
6322                case 8: /* D1 page flip */
6323                case 10: /* D2 page flip */
6324                case 12: /* D3 page flip */
6325                case 14: /* D4 page flip */
6326                case 16: /* D5 page flip */
6327                case 18: /* D6 page flip */
6328                        DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6329                        if (radeon_use_pflipirq > 0)
6330                                radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6331                        break;
6332                case 42: /* HPD hotplug */
6333                        if (src_data <= 5) {
6334                                hpd_idx = src_data;
6335                                mask = DC_HPD1_INTERRUPT;
6336                                queue_hotplug = true;
6337                                event_name = "HPD";
6338
6339                        } else if (src_data <= 11) {
6340                                hpd_idx = src_data - 6;
6341                                mask = DC_HPD1_RX_INTERRUPT;
6342                                queue_dp = true;
6343                                event_name = "HPD_RX";
6344
6345                        } else {
6346                                DRM_DEBUG("Unhandled interrupt: %d %d\n",
6347                                          src_id, src_data);
6348                                break;
6349                        }
6350
6351                        if (!(disp_int[hpd_idx] & mask))
6352                                DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6353
6354                        disp_int[hpd_idx] &= ~mask;
6355                        DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
6356                        break;
6357                case 96:
6358                        DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6359                        WREG32(SRBM_INT_ACK, 0x1);
6360                        break;
6361                case 124: /* UVD */
6362                        DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6363                        radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6364                        break;
6365                case 146:
6366                case 147:
6367                        addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6368                        status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6369                        /* reset addr and status */
6370                        WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6371                        if (addr == 0x0 && status == 0x0)
6372                                break;
6373                        dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6374                        dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
6375                                addr);
6376                        dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6377                                status);
6378                        si_vm_decode_fault(rdev, status, addr);
6379                        break;
6380                case 176: /* RINGID0 CP_INT */
6381                        radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6382                        break;
6383                case 177: /* RINGID1 CP_INT */
6384                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6385                        break;
6386                case 178: /* RINGID2 CP_INT */
6387                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6388                        break;
6389                case 181: /* CP EOP event */
6390                        DRM_DEBUG("IH: CP EOP\n");
6391                        switch (ring_id) {
6392                        case 0:
6393                                radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6394                                break;
6395                        case 1:
6396                                radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6397                                break;
6398                        case 2:
6399                                radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6400                                break;
6401                        }
6402                        break;
6403                case 224: /* DMA trap event */
6404                        DRM_DEBUG("IH: DMA trap\n");
6405                        radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6406                        break;
6407                case 230: /* thermal low to high */
6408                        DRM_DEBUG("IH: thermal low to high\n");
6409                        rdev->pm.dpm.thermal.high_to_low = false;
6410                        queue_thermal = true;
6411                        break;
6412                case 231: /* thermal high to low */
6413                        DRM_DEBUG("IH: thermal high to low\n");
6414                        rdev->pm.dpm.thermal.high_to_low = true;
6415                        queue_thermal = true;
6416                        break;
6417                case 233: /* GUI IDLE */
6418                        DRM_DEBUG("IH: GUI idle\n");
6419                        break;
6420                case 244: /* DMA trap event */
6421                        DRM_DEBUG("IH: DMA1 trap\n");
6422                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6423                        break;
6424                default:
6425                        DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6426                        break;
6427                }
6428
6429                /* wptr/rptr are in bytes! */
6430                rptr += 16;
6431                rptr &= rdev->ih.ptr_mask;
6432                WREG32(IH_RB_RPTR, rptr);
6433        }
6434        if (queue_dp)
6435                schedule_work(&rdev->dp_work);
6436        if (queue_hotplug)
6437                schedule_delayed_work(&rdev->hotplug_work, 0);
6438        if (queue_thermal && rdev->pm.dpm_enabled)
6439                schedule_work(&rdev->pm.dpm.thermal.work);
6440        rdev->ih.rptr = rptr;
6441        atomic_set(&rdev->ih.lock, 0);
6442
6443        /* make sure wptr hasn't changed while processing */
6444        wptr = si_get_ih_wptr(rdev);
6445        if (wptr != rptr)
6446                goto restart_ih;
6447
6448        return IRQ_HANDLED;
6449}
6450
6451/*
6452 * startup/shutdown callbacks
6453 */
6454static void si_uvd_init(struct radeon_device *rdev)
6455{
6456        int r;
6457
6458        if (!rdev->has_uvd)
6459                return;
6460
6461        r = radeon_uvd_init(rdev);
6462        if (r) {
6463                dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
6464                /*
6465                 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
6466                 * to early fails uvd_v2_2_resume() and thus nothing happens
6467                 * there. So it is pointless to try to go through that code
6468                 * hence why we disable uvd here.
6469                 */
6470                rdev->has_uvd = false;
6471                return;
6472        }
6473        rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
6474        r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
6475}
6476
6477static void si_uvd_start(struct radeon_device *rdev)
6478{
6479        int r;
6480
6481        if (!rdev->has_uvd)
6482                return;
6483
6484        r = uvd_v2_2_resume(rdev);
6485        if (r) {
6486                dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
6487                goto error;
6488        }
6489        r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
6490        if (r) {
6491                dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
6492                goto error;
6493        }
6494        return;
6495
6496error:
6497        rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6498}
6499
6500static void si_uvd_resume(struct radeon_device *rdev)
6501{
6502        struct radeon_ring *ring;
6503        int r;
6504
6505        if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
6506                return;
6507
6508        ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6509        r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
6510        if (r) {
6511                dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
6512                return;
6513        }
6514        r = uvd_v1_0_init(rdev);
6515        if (r) {
6516                dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
6517                return;
6518        }
6519}
6520
6521static void si_vce_init(struct radeon_device *rdev)
6522{
6523        int r;
6524
6525        if (!rdev->has_vce)
6526                return;
6527
6528        r = radeon_vce_init(rdev);
6529        if (r) {
6530                dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
6531                /*
6532                 * At this point rdev->vce.vcpu_bo is NULL which trickles down
6533                 * to early fails si_vce_start() and thus nothing happens
6534                 * there. So it is pointless to try to go through that code
6535                 * hence why we disable vce here.
6536                 */
6537                rdev->has_vce = false;
6538                return;
6539        }
6540        rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
6541        r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
6542        rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
6543        r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
6544}
6545
6546static void si_vce_start(struct radeon_device *rdev)
6547{
6548        int r;
6549
6550        if (!rdev->has_vce)
6551                return;
6552
6553        r = radeon_vce_resume(rdev);
6554        if (r) {
6555                dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6556                goto error;
6557        }
6558        r = vce_v1_0_resume(rdev);
6559        if (r) {
6560                dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6561                goto error;
6562        }
6563        r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
6564        if (r) {
6565                dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
6566                goto error;
6567        }
6568        r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
6569        if (r) {
6570                dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
6571                goto error;
6572        }
6573        return;
6574
6575error:
6576        rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6577        rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6578}
6579
6580static void si_vce_resume(struct radeon_device *rdev)
6581{
6582        struct radeon_ring *ring;
6583        int r;
6584
6585        if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
6586                return;
6587
6588        ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
6589        r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6590        if (r) {
6591                dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6592                return;
6593        }
6594        ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
6595        r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6596        if (r) {
6597                dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6598                return;
6599        }
6600        r = vce_v1_0_init(rdev);
6601        if (r) {
6602                dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
6603                return;
6604        }
6605}
6606
6607static int si_startup(struct radeon_device *rdev)
6608{
6609        struct radeon_ring *ring;
6610        int r;
6611
6612        /* enable pcie gen2/3 link */
6613        si_pcie_gen3_enable(rdev);
6614        /* enable aspm */
6615        si_program_aspm(rdev);
6616
6617        /* scratch needs to be initialized before MC */
6618        r = r600_vram_scratch_init(rdev);
6619        if (r)
6620                return r;
6621
6622        si_mc_program(rdev);
6623
6624        if (!rdev->pm.dpm_enabled) {
6625                r = si_mc_load_microcode(rdev);
6626                if (r) {
6627                        DRM_ERROR("Failed to load MC firmware!\n");
6628                        return r;
6629                }
6630        }
6631
6632        r = si_pcie_gart_enable(rdev);
6633        if (r)
6634                return r;
6635        si_gpu_init(rdev);
6636
6637        /* allocate rlc buffers */
6638        if (rdev->family == CHIP_VERDE) {
6639                rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6640                rdev->rlc.reg_list_size =
6641                        (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6642        }
6643        rdev->rlc.cs_data = si_cs_data;
6644        r = sumo_rlc_init(rdev);
6645        if (r) {
6646                DRM_ERROR("Failed to init rlc BOs!\n");
6647                return r;
6648        }
6649
6650        /* allocate wb buffer */
6651        r = radeon_wb_init(rdev);
6652        if (r)
6653                return r;
6654
6655        r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6656        if (r) {
6657                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6658                return r;
6659        }
6660
6661        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6662        if (r) {
6663                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6664                return r;
6665        }
6666
6667        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6668        if (r) {
6669                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6670                return r;
6671        }
6672
6673        r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6674        if (r) {
6675                dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6676                return r;
6677        }
6678
6679        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6680        if (r) {
6681                dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6682                return r;
6683        }
6684
6685        si_uvd_start(rdev);
6686        si_vce_start(rdev);
6687
6688        /* Enable IRQ */
6689        if (!rdev->irq.installed) {
6690                r = radeon_irq_kms_init(rdev);
6691                if (r)
6692                        return r;
6693        }
6694
6695        r = si_irq_init(rdev);
6696        if (r) {
6697                DRM_ERROR("radeon: IH init failed (%d).\n", r);
6698                radeon_irq_kms_fini(rdev);
6699                return r;
6700        }
6701        si_irq_set(rdev);
6702
6703        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6704        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6705                             RADEON_CP_PACKET2);
6706        if (r)
6707                return r;
6708
6709        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6710        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6711                             RADEON_CP_PACKET2);
6712        if (r)
6713                return r;
6714
6715        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6716        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6717                             RADEON_CP_PACKET2);
6718        if (r)
6719                return r;
6720
6721        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6722        r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6723                             DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6724        if (r)
6725                return r;
6726
6727        ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6728        r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6729                             DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6730        if (r)
6731                return r;
6732
6733        r = si_cp_load_microcode(rdev);
6734        if (r)
6735                return r;
6736        r = si_cp_resume(rdev);
6737        if (r)
6738                return r;
6739
6740        r = cayman_dma_resume(rdev);
6741        if (r)
6742                return r;
6743
6744        si_uvd_resume(rdev);
6745        si_vce_resume(rdev);
6746
6747        r = radeon_ib_pool_init(rdev);
6748        if (r) {
6749                dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6750                return r;
6751        }
6752
6753        r = radeon_vm_manager_init(rdev);
6754        if (r) {
6755                dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6756                return r;
6757        }
6758
6759        r = radeon_audio_init(rdev);
6760        if (r)
6761                return r;
6762
6763        return 0;
6764}
6765
6766int si_resume(struct radeon_device *rdev)
6767{
6768        int r;
6769
6770        /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6771         * posting will perform necessary task to bring back GPU into good
6772         * shape.
6773         */
6774        /* post card */
6775        atom_asic_init(rdev->mode_info.atom_context);
6776
6777        /* init golden registers */
6778        si_init_golden_registers(rdev);
6779
6780        if (rdev->pm.pm_method == PM_METHOD_DPM)
6781                radeon_pm_resume(rdev);
6782
6783        rdev->accel_working = true;
6784        r = si_startup(rdev);
6785        if (r) {
6786                DRM_ERROR("si startup failed on resume\n");
6787                rdev->accel_working = false;
6788                return r;
6789        }
6790
6791        return r;
6792
6793}
6794
6795int si_suspend(struct radeon_device *rdev)
6796{
6797        radeon_pm_suspend(rdev);
6798        radeon_audio_fini(rdev);
6799        radeon_vm_manager_fini(rdev);
6800        si_cp_enable(rdev, false);
6801        cayman_dma_stop(rdev);
6802        if (rdev->has_uvd) {
6803                uvd_v1_0_fini(rdev);
6804                radeon_uvd_suspend(rdev);
6805        }
6806        if (rdev->has_vce)
6807                radeon_vce_suspend(rdev);
6808        si_fini_pg(rdev);
6809        si_fini_cg(rdev);
6810        si_irq_suspend(rdev);
6811        radeon_wb_disable(rdev);
6812        si_pcie_gart_disable(rdev);
6813        return 0;
6814}
6815
6816/* Plan is to move initialization in that function and use
6817 * helper function so that radeon_device_init pretty much
6818 * do nothing more than calling asic specific function. This
6819 * should also allow to remove a bunch of callback function
6820 * like vram_info.
6821 */
6822int si_init(struct radeon_device *rdev)
6823{
6824        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6825        int r;
6826
6827        /* Read BIOS */
6828        if (!radeon_get_bios(rdev)) {
6829                if (ASIC_IS_AVIVO(rdev))
6830                        return -EINVAL;
6831        }
6832        /* Must be an ATOMBIOS */
6833        if (!rdev->is_atom_bios) {
6834                dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6835                return -EINVAL;
6836        }
6837        r = radeon_atombios_init(rdev);
6838        if (r)
6839                return r;
6840
6841        /* Post card if necessary */
6842        if (!radeon_card_posted(rdev)) {
6843                if (!rdev->bios) {
6844                        dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6845                        return -EINVAL;
6846                }
6847                DRM_INFO("GPU not posted. posting now...\n");
6848                atom_asic_init(rdev->mode_info.atom_context);
6849        }
6850        /* init golden registers */
6851        si_init_golden_registers(rdev);
6852        /* Initialize scratch registers */
6853        si_scratch_init(rdev);
6854        /* Initialize surface registers */
6855        radeon_surface_init(rdev);
6856        /* Initialize clocks */
6857        radeon_get_clock_info(rdev->ddev);
6858
6859        /* Fence driver */
6860        radeon_fence_driver_init(rdev);
6861
6862        /* initialize memory controller */
6863        r = si_mc_init(rdev);
6864        if (r)
6865                return r;
6866        /* Memory manager */
6867        r = radeon_bo_init(rdev);
6868        if (r)
6869                return r;
6870
6871        if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6872            !rdev->rlc_fw || !rdev->mc_fw) {
6873                r = si_init_microcode(rdev);
6874                if (r) {
6875                        DRM_ERROR("Failed to load firmware!\n");
6876                        return r;
6877                }
6878        }
6879
6880        /* Initialize power management */
6881        radeon_pm_init(rdev);
6882
6883        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6884        ring->ring_obj = NULL;
6885        r600_ring_init(rdev, ring, 1024 * 1024);
6886
6887        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6888        ring->ring_obj = NULL;
6889        r600_ring_init(rdev, ring, 1024 * 1024);
6890
6891        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6892        ring->ring_obj = NULL;
6893        r600_ring_init(rdev, ring, 1024 * 1024);
6894
6895        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6896        ring->ring_obj = NULL;
6897        r600_ring_init(rdev, ring, 64 * 1024);
6898
6899        ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6900        ring->ring_obj = NULL;
6901        r600_ring_init(rdev, ring, 64 * 1024);
6902
6903        si_uvd_init(rdev);
6904        si_vce_init(rdev);
6905
6906        rdev->ih.ring_obj = NULL;
6907        r600_ih_ring_init(rdev, 64 * 1024);
6908
6909        r = r600_pcie_gart_init(rdev);
6910        if (r)
6911                return r;
6912
6913        rdev->accel_working = true;
6914        r = si_startup(rdev);
6915        if (r) {
6916                dev_err(rdev->dev, "disabling GPU acceleration\n");
6917                si_cp_fini(rdev);
6918                cayman_dma_fini(rdev);
6919                si_irq_fini(rdev);
6920                sumo_rlc_fini(rdev);
6921                radeon_wb_fini(rdev);
6922                radeon_ib_pool_fini(rdev);
6923                radeon_vm_manager_fini(rdev);
6924                radeon_irq_kms_fini(rdev);
6925                si_pcie_gart_fini(rdev);
6926                rdev->accel_working = false;
6927        }
6928
6929        /* Don't start up if the MC ucode is missing.
6930         * The default clocks and voltages before the MC ucode
6931         * is loaded are not suffient for advanced operations.
6932         */
6933        if (!rdev->mc_fw) {
6934                DRM_ERROR("radeon: MC ucode required for NI+.\n");
6935                return -EINVAL;
6936        }
6937
6938        return 0;
6939}
6940
6941void si_fini(struct radeon_device *rdev)
6942{
6943        radeon_pm_fini(rdev);
6944        si_cp_fini(rdev);
6945        cayman_dma_fini(rdev);
6946        si_fini_pg(rdev);
6947        si_fini_cg(rdev);
6948        si_irq_fini(rdev);
6949        sumo_rlc_fini(rdev);
6950        radeon_wb_fini(rdev);
6951        radeon_vm_manager_fini(rdev);
6952        radeon_ib_pool_fini(rdev);
6953        radeon_irq_kms_fini(rdev);
6954        if (rdev->has_uvd) {
6955                uvd_v1_0_fini(rdev);
6956                radeon_uvd_fini(rdev);
6957        }
6958        if (rdev->has_vce)
6959                radeon_vce_fini(rdev);
6960        si_pcie_gart_fini(rdev);
6961        r600_vram_scratch_fini(rdev);
6962        radeon_gem_fini(rdev);
6963        radeon_fence_driver_fini(rdev);
6964        radeon_bo_fini(rdev);
6965        radeon_atombios_fini(rdev);
6966        kfree(rdev->bios);
6967        rdev->bios = NULL;
6968}
6969
6970/**
6971 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6972 *
6973 * @rdev: radeon_device pointer
6974 *
6975 * Fetches a GPU clock counter snapshot (SI).
6976 * Returns the 64 bit clock counter snapshot.
6977 */
6978uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6979{
6980        uint64_t clock;
6981
6982        mutex_lock(&rdev->gpu_clock_mutex);
6983        WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6984        clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6985                ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6986        mutex_unlock(&rdev->gpu_clock_mutex);
6987        return clock;
6988}
6989
6990int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6991{
6992        unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6993        int r;
6994
6995        /* bypass vclk and dclk with bclk */
6996        WREG32_P(CG_UPLL_FUNC_CNTL_2,
6997                VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6998                ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6999
7000        /* put PLL in bypass mode */

7001        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7002
7003        if (!vclk || !dclk) {
7004                /* keep the Bypass mode */
7005                return 0;
7006        }
7007
7008        r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7009                                          16384, 0x03FFFFFF, 0, 128, 5,
7010                                          &fb_div, &vclk_div, &dclk_div);
7011        if (r)
7012                return r;
7013
7014        /* set RESET_ANTI_MUX to 0 */
7015        WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7016
7017        /* set VCO_MODE to 1 */
7018        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7019
7020        /* disable sleep mode */
7021        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7022
7023        /* deassert UPLL_RESET */
7024        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7025
7026        mdelay(1);
7027
7028        r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7029        if (r)
7030                return r;
7031
7032        /* assert UPLL_RESET again */
7033        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7034
7035        /* disable spread spectrum. */
7036        WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7037
7038        /* set feedback divider */
7039        WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
7040
7041        /* set ref divider to 0 */
7042        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7043
7044        if (fb_div < 307200)
7045                WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7046        else
7047                WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7048
7049        /* set PDIV_A and PDIV_B */
7050        WREG32_P(CG_UPLL_FUNC_CNTL_2,
7051                UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
7052                ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7053
7054        /* give the PLL some time to settle */
7055        mdelay(15);
7056
7057        /* deassert PLL_RESET */
7058        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7059
7060        mdelay(15);
7061
7062        /* switch from bypass mode to normal mode */
7063        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7064
7065        r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7066        if (r)
7067                return r;
7068
7069        /* switch VCLK and DCLK selection */
7070        WREG32_P(CG_UPLL_FUNC_CNTL_2,
7071                VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7072                ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7073
7074        mdelay(100);
7075
7076        return 0;
7077}
7078
7079static void si_pcie_gen3_enable(struct radeon_device *rdev)
7080{
7081        struct pci_dev *root = rdev->pdev->bus->self;
7082        enum pci_bus_speed speed_cap;
7083        u32 speed_cntl, current_data_rate;
7084        int i;
7085        u16 tmp16;
7086
7087        if (pci_is_root_bus(rdev->pdev->bus))
7088                return;
7089
7090        if (radeon_pcie_gen2 == 0)
7091                return;
7092
7093        if (rdev->flags & RADEON_IS_IGP)
7094                return;
7095
7096        if (!(rdev->flags & RADEON_IS_PCIE))
7097                return;
7098
7099        speed_cap = pcie_get_speed_cap(root);
7100        if (speed_cap == PCI_SPEED_UNKNOWN)
7101                return;
7102
7103        if ((speed_cap != PCIE_SPEED_8_0GT) &&
7104            (speed_cap != PCIE_SPEED_5_0GT))
7105                return;
7106
7107        speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7108        current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7109                LC_CURRENT_DATA_RATE_SHIFT;
7110        if (speed_cap == PCIE_SPEED_8_0GT) {
7111                if (current_data_rate == 2) {
7112                        DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7113                        return;
7114                }
7115                DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7116        } else if (speed_cap == PCIE_SPEED_5_0GT) {
7117                if (current_data_rate == 1) {
7118                        DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7119                        return;
7120                }
7121                DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7122        }
7123
7124        if (!pci_is_pcie(root) || !pci_is_pcie(rdev->pdev))
7125                return;
7126
7127        if (speed_cap == PCIE_SPEED_8_0GT) {
7128                /* re-try equalization if gen3 is not already enabled */
7129                if (current_data_rate != 2) {
7130                        u16 bridge_cfg, gpu_cfg;
7131                        u16 bridge_cfg2, gpu_cfg2;
7132                        u32 max_lw, current_lw, tmp;
7133
7134                        pcie_capability_read_word(root, PCI_EXP_LNKCTL,
7135                                                  &bridge_cfg);
7136                        pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL,
7137                                                  &gpu_cfg);
7138
7139                        tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7140                        pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
7141
7142                        tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7143                        pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL,
7144                                                   tmp16);
7145
7146                        tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7147                        max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7148                        current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7149
7150                        if (current_lw < max_lw) {
7151                                tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7152                                if (tmp & LC_RENEGOTIATION_SUPPORT) {
7153                                        tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7154                                        tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7155                                        tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7156                                        WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7157                                }
7158                        }
7159
7160                        for (i = 0; i < 10; i++) {
7161                                /* check status */
7162                                pcie_capability_read_word(rdev->pdev,
7163                                                          PCI_EXP_DEVSTA,
7164                                                          &tmp16);
7165                                if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7166                                        break;
7167
7168                                pcie_capability_read_word(root, PCI_EXP_LNKCTL,
7169                                                          &bridge_cfg);
7170                                pcie_capability_read_word(rdev->pdev,
7171                                                          PCI_EXP_LNKCTL,
7172                                                          &gpu_cfg);
7173
7174                                pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
7175                                                          &bridge_cfg2);
7176                                pcie_capability_read_word(rdev->pdev,
7177                                                          PCI_EXP_LNKCTL2,
7178                                                          &gpu_cfg2);
7179
7180                                tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7181                                tmp |= LC_SET_QUIESCE;
7182                                WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7183
7184                                tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7185                                tmp |= LC_REDO_EQ;
7186                                WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7187
7188                                msleep(100);
7189
7190                                /* linkctl */
7191                                pcie_capability_read_word(root, PCI_EXP_LNKCTL,
7192                                                          &tmp16);
7193                                tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7194                                tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7195                                pcie_capability_write_word(root,
7196                                                           PCI_EXP_LNKCTL,
7197                                                           tmp16);
7198
7199                                pcie_capability_read_word(rdev->pdev,
7200                                                          PCI_EXP_LNKCTL,
7201                                                          &tmp16);
7202                                tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7203                                tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7204                                pcie_capability_write_word(rdev->pdev,
7205                                                           PCI_EXP_LNKCTL,
7206                                                           tmp16);
7207
7208                                /* linkctl2 */
7209                                pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
7210                                                          &tmp16);
7211                                tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
7212                                           PCI_EXP_LNKCTL2_TX_MARGIN);
7213                                tmp16 |= (bridge_cfg2 &
7214                                          (PCI_EXP_LNKCTL2_ENTER_COMP |
7215                                           PCI_EXP_LNKCTL2_TX_MARGIN));
7216                                pcie_capability_write_word(root,
7217                                                           PCI_EXP_LNKCTL2,
7218                                                           tmp16);
7219
7220                                pcie_capability_read_word(rdev->pdev,
7221                                                          PCI_EXP_LNKCTL2,
7222                                                          &tmp16);
7223                                tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
7224                                           PCI_EXP_LNKCTL2_TX_MARGIN);
7225                                tmp16 |= (gpu_cfg2 &
7226                                          (PCI_EXP_LNKCTL2_ENTER_COMP |
7227                                           PCI_EXP_LNKCTL2_TX_MARGIN));
7228                                pcie_capability_write_word(rdev->pdev,
7229                                                           PCI_EXP_LNKCTL2,
7230                                                           tmp16);
7231
7232                                tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7233                                tmp &= ~LC_SET_QUIESCE;
7234                                WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7235                        }
7236                }
7237        }
7238
7239        /* set the link speed */
7240        speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7241        speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7242        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7243
7244        pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL2, &tmp16);
7245        tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
7246        if (speed_cap == PCIE_SPEED_8_0GT)
7247                tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
7248        else if (speed_cap == PCIE_SPEED_5_0GT)
7249                tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
7250        else
7251                tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
7252        pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL2, tmp16);
7253
7254        speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7255        speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7256        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7257
7258        for (i = 0; i < rdev->usec_timeout; i++) {
7259                speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7260                if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7261                        break;
7262                udelay(1);
7263        }
7264}
7265
7266static void si_program_aspm(struct radeon_device *rdev)
7267{
7268        u32 data, orig;
7269        bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7270        bool disable_clkreq = false;
7271
7272        if (radeon_aspm == 0)
7273                return;
7274
7275        if (!(rdev->flags & RADEON_IS_PCIE))
7276                return;
7277
7278        orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7279        data &= ~LC_XMIT_N_FTS_MASK;
7280        data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7281        if (orig != data)
7282                WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7283
7284        orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7285        data |= LC_GO_TO_RECOVERY;
7286        if (orig != data)
7287                WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7288
7289        orig = data = RREG32_PCIE(PCIE_P_CNTL);
7290        data |= P_IGNORE_EDB_ERR;
7291        if (orig != data)
7292                WREG32_PCIE(PCIE_P_CNTL, data);
7293
7294        orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7295        data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7296        data |= LC_PMI_TO_L1_DIS;
7297        if (!disable_l0s)
7298                data |= LC_L0S_INACTIVITY(7);
7299
7300        if (!disable_l1) {
7301                data |= LC_L1_INACTIVITY(7);
7302                data &= ~LC_PMI_TO_L1_DIS;
7303                if (orig != data)
7304                        WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7305
7306                if (!disable_plloff_in_l1) {
7307                        bool clk_req_support;
7308
7309                        orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7310                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7311                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7312                        if (orig != data)
7313                                WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7314
7315                        orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7316                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7317                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7318                        if (orig != data)
7319                                WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7320
7321                        orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7322                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7323                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7324                        if (orig != data)
7325                                WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7326
7327                        orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7328                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7329                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7330                        if (orig != data)
7331                                WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7332
7333                        if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7334                                orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7335                                data &= ~PLL_RAMP_UP_TIME_0_MASK;
7336                                if (orig != data)
7337                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7338
7339                                orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7340                                data &= ~PLL_RAMP_UP_TIME_1_MASK;
7341                                if (orig != data)
7342                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7343
7344                                orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7345                                data &= ~PLL_RAMP_UP_TIME_2_MASK;
7346                                if (orig != data)
7347                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7348
7349                                orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7350                                data &= ~PLL_RAMP_UP_TIME_3_MASK;
7351                                if (orig != data)
7352                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7353
7354                                orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7355                                data &= ~PLL_RAMP_UP_TIME_0_MASK;
7356                                if (orig != data)
7357                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7358
7359                                orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7360                                data &= ~PLL_RAMP_UP_TIME_1_MASK;
7361                                if (orig != data)
7362                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7363
7364                                orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7365                                data &= ~PLL_RAMP_UP_TIME_2_MASK;
7366                                if (orig != data)
7367                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7368
7369                                orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7370                                data &= ~PLL_RAMP_UP_TIME_3_MASK;
7371                                if (orig != data)
7372                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7373                        }
7374                        orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7375                        data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7376                        data |= LC_DYN_LANES_PWR_STATE(3);
7377                        if (orig != data)
7378                                WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7379
7380                        orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7381                        data &= ~LS2_EXIT_TIME_MASK;
7382                        if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7383                                data |= LS2_EXIT_TIME(5);
7384                        if (orig != data)
7385                                WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7386
7387                        orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7388                        data &= ~LS2_EXIT_TIME_MASK;
7389                        if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7390                                data |= LS2_EXIT_TIME(5);
7391                        if (orig != data)
7392                                WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7393
7394                        if (!disable_clkreq &&
7395                            !pci_is_root_bus(rdev->pdev->bus)) {
7396                                struct pci_dev *root = rdev->pdev->bus->self;
7397                                u32 lnkcap;
7398
7399                                clk_req_support = false;
7400                                pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7401                                if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7402                                        clk_req_support = true;
7403                        } else {
7404                                clk_req_support = false;
7405                        }
7406
7407                        if (clk_req_support) {
7408                                orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7409                                data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7410                                if (orig != data)
7411                                        WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7412
7413                                orig = data = RREG32(THM_CLK_CNTL);
7414                                data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7415                                data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7416                                if (orig != data)
7417                                        WREG32(THM_CLK_CNTL, data);
7418
7419                                orig = data = RREG32(MISC_CLK_CNTL);
7420                                data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7421                                data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7422                                if (orig != data)
7423                                        WREG32(MISC_CLK_CNTL, data);
7424
7425                                orig = data = RREG32(CG_CLKPIN_CNTL);
7426                                data &= ~BCLK_AS_XCLK;
7427                                if (orig != data)
7428                                        WREG32(CG_CLKPIN_CNTL, data);
7429
7430                                orig = data = RREG32(CG_CLKPIN_CNTL_2);
7431                                data &= ~FORCE_BIF_REFCLK_EN;
7432                                if (orig != data)
7433                                        WREG32(CG_CLKPIN_CNTL_2, data);
7434
7435                                orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7436                                data &= ~MPLL_CLKOUT_SEL_MASK;
7437                                data |= MPLL_CLKOUT_SEL(4);
7438                                if (orig != data)
7439                                        WREG32(MPLL_BYPASSCLK_SEL, data);
7440
7441                                orig = data = RREG32(SPLL_CNTL_MODE);
7442                                data &= ~SPLL_REFCLK_SEL_MASK;
7443                                if (orig != data)
7444                                        WREG32(SPLL_CNTL_MODE, data);
7445                        }
7446                }
7447        } else {
7448                if (orig != data)
7449                        WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7450        }
7451
7452        orig = data = RREG32_PCIE(PCIE_CNTL2);
7453        data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7454        if (orig != data)
7455                WREG32_PCIE(PCIE_CNTL2, data);
7456
7457        if (!disable_l0s) {
7458                data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7459                if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7460                        data = RREG32_PCIE(PCIE_LC_STATUS1);
7461                        if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7462                                orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7463                                data &= ~LC_L0S_INACTIVITY_MASK;
7464                                if (orig != data)
7465                                        WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7466                        }
7467                }
7468        }
7469}
7470
7471static int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
7472{
7473        unsigned i;
7474
7475        /* make sure VCEPLL_CTLREQ is deasserted */
7476        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7477
7478        mdelay(10);
7479
7480        /* assert UPLL_CTLREQ */
7481        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
7482
7483        /* wait for CTLACK and CTLACK2 to get asserted */
7484        for (i = 0; i < 100; ++i) {
7485                uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7486                if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7487                        break;
7488                mdelay(10);
7489        }
7490
7491        /* deassert UPLL_CTLREQ */
7492        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7493
7494        if (i == 100) {
7495                DRM_ERROR("Timeout setting UVD clocks!\n");
7496                return -ETIMEDOUT;
7497        }
7498
7499        return 0;
7500}
7501
7502int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7503{
7504        unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7505        int r;
7506
7507        /* bypass evclk and ecclk with bclk */
7508        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7509                     EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7510                     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7511
7512        /* put PLL in bypass mode */
7513        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7514                     ~VCEPLL_BYPASS_EN_MASK);
7515
7516        if (!evclk || !ecclk) {
7517                /* keep the Bypass mode, put PLL to sleep */
7518                WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7519                             ~VCEPLL_SLEEP_MASK);
7520                return 0;
7521        }
7522
7523        r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7524                                          16384, 0x03FFFFFF, 0, 128, 5,
7525                                          &fb_div, &evclk_div, &ecclk_div);
7526        if (r)
7527                return r;
7528
7529        /* set RESET_ANTI_MUX to 0 */
7530        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7531
7532        /* set VCO_MODE to 1 */
7533        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7534                     ~VCEPLL_VCO_MODE_MASK);
7535
7536        /* toggle VCEPLL_SLEEP to 1 then back to 0 */
7537        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7538                     ~VCEPLL_SLEEP_MASK);
7539        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7540
7541        /* deassert VCEPLL_RESET */
7542        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7543
7544        mdelay(1);
7545
7546        r = si_vce_send_vcepll_ctlreq(rdev);
7547        if (r)
7548                return r;
7549
7550        /* assert VCEPLL_RESET again */
7551        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7552
7553        /* disable spread spectrum. */
7554        WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7555
7556        /* set feedback divider */
7557        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7558
7559        /* set ref divider to 0 */
7560        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7561
7562        /* set PDIV_A and PDIV_B */
7563        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7564                     VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7565                     ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7566
7567        /* give the PLL some time to settle */
7568        mdelay(15);
7569
7570        /* deassert PLL_RESET */
7571        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7572
7573        mdelay(15);
7574
7575        /* switch from bypass mode to normal mode */
7576        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7577
7578        r = si_vce_send_vcepll_ctlreq(rdev);
7579        if (r)
7580                return r;
7581
7582        /* switch VCLK and DCLK selection */
7583        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7584                     EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7585                     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7586
7587        mdelay(100);
7588
7589        return 0;
7590}
7591