LXR linux/drivers/gpu/drm/radeon/si.c

   1/*
   2 * Copyright 2011 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24#include <linux/firmware.h>
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include <drm/drmP.h>
  28#include "radeon.h"
  29#include "radeon_asic.h"
  30#include "radeon_audio.h"
  31#include <drm/radeon_drm.h>
  32#include "sid.h"
  33#include "atom.h"
  34#include "si_blit_shaders.h"
  35#include "clearstate_si.h"
  36#include "radeon_ucode.h"
  37
  38
  39MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
  40MODULE_FIRMWARE("radeon/TAHITI_me.bin");
  41MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
  42MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
  43MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
  44MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
  45MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
  46
  47MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
  48MODULE_FIRMWARE("radeon/tahiti_me.bin");
  49MODULE_FIRMWARE("radeon/tahiti_ce.bin");
  50MODULE_FIRMWARE("radeon/tahiti_mc.bin");
  51MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
  52MODULE_FIRMWARE("radeon/tahiti_smc.bin");
  53
  54MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
  55MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
  56MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
  57MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
  58MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
  59MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
  60MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
  61
  62MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
  63MODULE_FIRMWARE("radeon/pitcairn_me.bin");
  64MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
  65MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
  66MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
  67MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
  68
  69MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
  70MODULE_FIRMWARE("radeon/VERDE_me.bin");
  71MODULE_FIRMWARE("radeon/VERDE_ce.bin");
  72MODULE_FIRMWARE("radeon/VERDE_mc.bin");
  73MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
  74MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
  75MODULE_FIRMWARE("radeon/VERDE_smc.bin");
  76
  77MODULE_FIRMWARE("radeon/verde_pfp.bin");
  78MODULE_FIRMWARE("radeon/verde_me.bin");
  79MODULE_FIRMWARE("radeon/verde_ce.bin");
  80MODULE_FIRMWARE("radeon/verde_mc.bin");
  81MODULE_FIRMWARE("radeon/verde_rlc.bin");
  82MODULE_FIRMWARE("radeon/verde_smc.bin");
  83
  84MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
  85MODULE_FIRMWARE("radeon/OLAND_me.bin");
  86MODULE_FIRMWARE("radeon/OLAND_ce.bin");
  87MODULE_FIRMWARE("radeon/OLAND_mc.bin");
  88MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
  89MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
  90MODULE_FIRMWARE("radeon/OLAND_smc.bin");
  91
  92MODULE_FIRMWARE("radeon/oland_pfp.bin");
  93MODULE_FIRMWARE("radeon/oland_me.bin");
  94MODULE_FIRMWARE("radeon/oland_ce.bin");
  95MODULE_FIRMWARE("radeon/oland_mc.bin");
  96MODULE_FIRMWARE("radeon/oland_rlc.bin");
  97MODULE_FIRMWARE("radeon/oland_smc.bin");
  98
  99MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
 100MODULE_FIRMWARE("radeon/HAINAN_me.bin");
 101MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
 102MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
 103MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
 104MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
 105MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
 106
 107MODULE_FIRMWARE("radeon/hainan_pfp.bin");
 108MODULE_FIRMWARE("radeon/hainan_me.bin");
 109MODULE_FIRMWARE("radeon/hainan_ce.bin");
 110MODULE_FIRMWARE("radeon/hainan_mc.bin");
 111MODULE_FIRMWARE("radeon/hainan_rlc.bin");
 112MODULE_FIRMWARE("radeon/hainan_smc.bin");
 113
 114static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 115static void si_pcie_gen3_enable(struct radeon_device *rdev);
 116static void si_program_aspm(struct radeon_device *rdev);
 117extern void sumo_rlc_fini(struct radeon_device *rdev);
 118extern int sumo_rlc_init(struct radeon_device *rdev);
 119extern int r600_ih_ring_alloc(struct radeon_device *rdev);
 120extern void r600_ih_ring_fini(struct radeon_device *rdev);
 121extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 122extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 123extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 124extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
 125extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 126extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 127static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
 128                                         bool enable);
 129static void si_init_pg(struct radeon_device *rdev);
 130static void si_init_cg(struct radeon_device *rdev);
 131static void si_fini_pg(struct radeon_device *rdev);
 132static void si_fini_cg(struct radeon_device *rdev);
 133static void si_rlc_stop(struct radeon_device *rdev);
 134
 135static const u32 verde_rlc_save_restore_register_list[] =
 136{
 137        (0x8000 << 16) | (0x98f4 >> 2),
 138        0x00000000,
 139        (0x8040 << 16) | (0x98f4 >> 2),
 140        0x00000000,
 141        (0x8000 << 16) | (0xe80 >> 2),
 142        0x00000000,
 143        (0x8040 << 16) | (0xe80 >> 2),
 144        0x00000000,
 145        (0x8000 << 16) | (0x89bc >> 2),
 146        0x00000000,
 147        (0x8040 << 16) | (0x89bc >> 2),
 148        0x00000000,
 149        (0x8000 << 16) | (0x8c1c >> 2),
 150        0x00000000,
 151        (0x8040 << 16) | (0x8c1c >> 2),
 152        0x00000000,
 153        (0x9c00 << 16) | (0x98f0 >> 2),
 154        0x00000000,
 155        (0x9c00 << 16) | (0xe7c >> 2),
 156        0x00000000,
 157        (0x8000 << 16) | (0x9148 >> 2),
 158        0x00000000,
 159        (0x8040 << 16) | (0x9148 >> 2),
 160        0x00000000,
 161        (0x9c00 << 16) | (0x9150 >> 2),
 162        0x00000000,
 163        (0x9c00 << 16) | (0x897c >> 2),
 164        0x00000000,
 165        (0x9c00 << 16) | (0x8d8c >> 2),
 166        0x00000000,
 167        (0x9c00 << 16) | (0xac54 >> 2),
 168        0X00000000,
 169        0x3,
 170        (0x9c00 << 16) | (0x98f8 >> 2),
 171        0x00000000,
 172        (0x9c00 << 16) | (0x9910 >> 2),
 173        0x00000000,
 174        (0x9c00 << 16) | (0x9914 >> 2),
 175        0x00000000,
 176        (0x9c00 << 16) | (0x9918 >> 2),
 177        0x00000000,
 178        (0x9c00 << 16) | (0x991c >> 2),
 179        0x00000000,
 180        (0x9c00 << 16) | (0x9920 >> 2),
 181        0x00000000,
 182        (0x9c00 << 16) | (0x9924 >> 2),
 183        0x00000000,
 184        (0x9c00 << 16) | (0x9928 >> 2),
 185        0x00000000,
 186        (0x9c00 << 16) | (0x992c >> 2),
 187        0x00000000,
 188        (0x9c00 << 16) | (0x9930 >> 2),
 189        0x00000000,
 190        (0x9c00 << 16) | (0x9934 >> 2),
 191        0x00000000,
 192        (0x9c00 << 16) | (0x9938 >> 2),
 193        0x00000000,
 194        (0x9c00 << 16) | (0x993c >> 2),
 195        0x00000000,
 196        (0x9c00 << 16) | (0x9940 >> 2),
 197        0x00000000,
 198        (0x9c00 << 16) | (0x9944 >> 2),
 199        0x00000000,
 200        (0x9c00 << 16) | (0x9948 >> 2),
 201        0x00000000,
 202        (0x9c00 << 16) | (0x994c >> 2),
 203        0x00000000,
 204        (0x9c00 << 16) | (0x9950 >> 2),
 205        0x00000000,
 206        (0x9c00 << 16) | (0x9954 >> 2),
 207        0x00000000,
 208        (0x9c00 << 16) | (0x9958 >> 2),
 209        0x00000000,
 210        (0x9c00 << 16) | (0x995c >> 2),
 211        0x00000000,
 212        (0x9c00 << 16) | (0x9960 >> 2),
 213        0x00000000,
 214        (0x9c00 << 16) | (0x9964 >> 2),
 215        0x00000000,
 216        (0x9c00 << 16) | (0x9968 >> 2),
 217        0x00000000,
 218        (0x9c00 << 16) | (0x996c >> 2),
 219        0x00000000,
 220        (0x9c00 << 16) | (0x9970 >> 2),
 221        0x00000000,
 222        (0x9c00 << 16) | (0x9974 >> 2),
 223        0x00000000,
 224        (0x9c00 << 16) | (0x9978 >> 2),
 225        0x00000000,
 226        (0x9c00 << 16) | (0x997c >> 2),
 227        0x00000000,
 228        (0x9c00 << 16) | (0x9980 >> 2),
 229        0x00000000,
 230        (0x9c00 << 16) | (0x9984 >> 2),
 231        0x00000000,
 232        (0x9c00 << 16) | (0x9988 >> 2),
 233        0x00000000,
 234        (0x9c00 << 16) | (0x998c >> 2),
 235        0x00000000,
 236        (0x9c00 << 16) | (0x8c00 >> 2),
 237        0x00000000,
 238        (0x9c00 << 16) | (0x8c14 >> 2),
 239        0x00000000,
 240        (0x9c00 << 16) | (0x8c04 >> 2),
 241        0x00000000,
 242        (0x9c00 << 16) | (0x8c08 >> 2),
 243        0x00000000,
 244        (0x8000 << 16) | (0x9b7c >> 2),
 245        0x00000000,
 246        (0x8040 << 16) | (0x9b7c >> 2),
 247        0x00000000,
 248        (0x8000 << 16) | (0xe84 >> 2),
 249        0x00000000,
 250        (0x8040 << 16) | (0xe84 >> 2),
 251        0x00000000,
 252        (0x8000 << 16) | (0x89c0 >> 2),
 253        0x00000000,
 254        (0x8040 << 16) | (0x89c0 >> 2),
 255        0x00000000,
 256        (0x8000 << 16) | (0x914c >> 2),
 257        0x00000000,
 258        (0x8040 << 16) | (0x914c >> 2),
 259        0x00000000,
 260        (0x8000 << 16) | (0x8c20 >> 2),
 261        0x00000000,
 262        (0x8040 << 16) | (0x8c20 >> 2),
 263        0x00000000,
 264        (0x8000 << 16) | (0x9354 >> 2),
 265        0x00000000,
 266        (0x8040 << 16) | (0x9354 >> 2),
 267        0x00000000,
 268        (0x9c00 << 16) | (0x9060 >> 2),
 269        0x00000000,
 270        (0x9c00 << 16) | (0x9364 >> 2),
 271        0x00000000,
 272        (0x9c00 << 16) | (0x9100 >> 2),
 273        0x00000000,
 274        (0x9c00 << 16) | (0x913c >> 2),
 275        0x00000000,
 276        (0x8000 << 16) | (0x90e0 >> 2),
 277        0x00000000,
 278        (0x8000 << 16) | (0x90e4 >> 2),
 279        0x00000000,
 280        (0x8000 << 16) | (0x90e8 >> 2),
 281        0x00000000,
 282        (0x8040 << 16) | (0x90e0 >> 2),
 283        0x00000000,
 284        (0x8040 << 16) | (0x90e4 >> 2),
 285        0x00000000,
 286        (0x8040 << 16) | (0x90e8 >> 2),
 287        0x00000000,
 288        (0x9c00 << 16) | (0x8bcc >> 2),
 289        0x00000000,
 290        (0x9c00 << 16) | (0x8b24 >> 2),
 291        0x00000000,
 292        (0x9c00 << 16) | (0x88c4 >> 2),
 293        0x00000000,
 294        (0x9c00 << 16) | (0x8e50 >> 2),
 295        0x00000000,
 296        (0x9c00 << 16) | (0x8c0c >> 2),
 297        0x00000000,
 298        (0x9c00 << 16) | (0x8e58 >> 2),
 299        0x00000000,
 300        (0x9c00 << 16) | (0x8e5c >> 2),
 301        0x00000000,
 302        (0x9c00 << 16) | (0x9508 >> 2),
 303        0x00000000,
 304        (0x9c00 << 16) | (0x950c >> 2),
 305        0x00000000,
 306        (0x9c00 << 16) | (0x9494 >> 2),
 307        0x00000000,
 308        (0x9c00 << 16) | (0xac0c >> 2),
 309        0x00000000,
 310        (0x9c00 << 16) | (0xac10 >> 2),
 311        0x00000000,
 312        (0x9c00 << 16) | (0xac14 >> 2),
 313        0x00000000,
 314        (0x9c00 << 16) | (0xae00 >> 2),
 315        0x00000000,
 316        (0x9c00 << 16) | (0xac08 >> 2),
 317        0x00000000,
 318        (0x9c00 << 16) | (0x88d4 >> 2),
 319        0x00000000,
 320        (0x9c00 << 16) | (0x88c8 >> 2),
 321        0x00000000,
 322        (0x9c00 << 16) | (0x88cc >> 2),
 323        0x00000000,
 324        (0x9c00 << 16) | (0x89b0 >> 2),
 325        0x00000000,
 326        (0x9c00 << 16) | (0x8b10 >> 2),
 327        0x00000000,
 328        (0x9c00 << 16) | (0x8a14 >> 2),
 329        0x00000000,
 330        (0x9c00 << 16) | (0x9830 >> 2),
 331        0x00000000,
 332        (0x9c00 << 16) | (0x9834 >> 2),
 333        0x00000000,
 334        (0x9c00 << 16) | (0x9838 >> 2),
 335        0x00000000,
 336        (0x9c00 << 16) | (0x9a10 >> 2),
 337        0x00000000,
 338        (0x8000 << 16) | (0x9870 >> 2),
 339        0x00000000,
 340        (0x8000 << 16) | (0x9874 >> 2),
 341        0x00000000,
 342        (0x8001 << 16) | (0x9870 >> 2),
 343        0x00000000,
 344        (0x8001 << 16) | (0x9874 >> 2),
 345        0x00000000,
 346        (0x8040 << 16) | (0x9870 >> 2),
 347        0x00000000,
 348        (0x8040 << 16) | (0x9874 >> 2),
 349        0x00000000,
 350        (0x8041 << 16) | (0x9870 >> 2),
 351        0x00000000,
 352        (0x8041 << 16) | (0x9874 >> 2),
 353        0x00000000,
 354        0x00000000
 355};
 356
 357static const u32 tahiti_golden_rlc_registers[] =
 358{
 359        0xc424, 0xffffffff, 0x00601005,
 360        0xc47c, 0xffffffff, 0x10104040,
 361        0xc488, 0xffffffff, 0x0100000a,
 362        0xc314, 0xffffffff, 0x00000800,
 363        0xc30c, 0xffffffff, 0x800000f4,
 364        0xf4a8, 0xffffffff, 0x00000000
 365};
 366
 367static const u32 tahiti_golden_registers[] =
 368{
 369        0x9a10, 0x00010000, 0x00018208,
 370        0x9830, 0xffffffff, 0x00000000,
 371        0x9834, 0xf00fffff, 0x00000400,
 372        0x9838, 0x0002021c, 0x00020200,
 373        0xc78, 0x00000080, 0x00000000,
 374        0xd030, 0x000300c0, 0x00800040,
 375        0xd830, 0x000300c0, 0x00800040,
 376        0x5bb0, 0x000000f0, 0x00000070,
 377        0x5bc0, 0x00200000, 0x50100000,
 378        0x7030, 0x31000311, 0x00000011,
 379        0x277c, 0x00000003, 0x000007ff,
 380        0x240c, 0x000007ff, 0x00000000,
 381        0x8a14, 0xf000001f, 0x00000007,
 382        0x8b24, 0xffffffff, 0x00ffffff,
 383        0x8b10, 0x0000ff0f, 0x00000000,
 384        0x28a4c, 0x07ffffff, 0x4e000000,
 385        0x28350, 0x3f3f3fff, 0x2a00126a,
 386        0x30, 0x000000ff, 0x0040,
 387        0x34, 0x00000040, 0x00004040,
 388        0x9100, 0x07ffffff, 0x03000000,
 389        0x8e88, 0x01ff1f3f, 0x00000000,
 390        0x8e84, 0x01ff1f3f, 0x00000000,
 391        0x9060, 0x0000007f, 0x00000020,
 392        0x9508, 0x00010000, 0x00010000,
 393        0xac14, 0x00000200, 0x000002fb,
 394        0xac10, 0xffffffff, 0x0000543b,
 395        0xac0c, 0xffffffff, 0xa9210876,
 396        0x88d0, 0xffffffff, 0x000fff40,
 397        0x88d4, 0x0000001f, 0x00000010,
 398        0x1410, 0x20000000, 0x20fffed8,
 399        0x15c0, 0x000c0fc0, 0x000c0400
 400};
 401
 402static const u32 tahiti_golden_registers2[] =
 403{
 404        0xc64, 0x00000001, 0x00000001
 405};
 406
 407static const u32 pitcairn_golden_rlc_registers[] =
 408{
 409        0xc424, 0xffffffff, 0x00601004,
 410        0xc47c, 0xffffffff, 0x10102020,
 411        0xc488, 0xffffffff, 0x01000020,
 412        0xc314, 0xffffffff, 0x00000800,
 413        0xc30c, 0xffffffff, 0x800000a4
 414};
 415
 416static const u32 pitcairn_golden_registers[] =
 417{
 418        0x9a10, 0x00010000, 0x00018208,
 419        0x9830, 0xffffffff, 0x00000000,
 420        0x9834, 0xf00fffff, 0x00000400,
 421        0x9838, 0x0002021c, 0x00020200,
 422        0xc78, 0x00000080, 0x00000000,
 423        0xd030, 0x000300c0, 0x00800040,
 424        0xd830, 0x000300c0, 0x00800040,
 425        0x5bb0, 0x000000f0, 0x00000070,
 426        0x5bc0, 0x00200000, 0x50100000,
 427        0x7030, 0x31000311, 0x00000011,
 428        0x2ae4, 0x00073ffe, 0x000022a2,
 429        0x240c, 0x000007ff, 0x00000000,
 430        0x8a14, 0xf000001f, 0x00000007,
 431        0x8b24, 0xffffffff, 0x00ffffff,
 432        0x8b10, 0x0000ff0f, 0x00000000,
 433        0x28a4c, 0x07ffffff, 0x4e000000,
 434        0x28350, 0x3f3f3fff, 0x2a00126a,
 435        0x30, 0x000000ff, 0x0040,
 436        0x34, 0x00000040, 0x00004040,
 437        0x9100, 0x07ffffff, 0x03000000,
 438        0x9060, 0x0000007f, 0x00000020,
 439        0x9508, 0x00010000, 0x00010000,
 440        0xac14, 0x000003ff, 0x000000f7,
 441        0xac10, 0xffffffff, 0x00000000,
 442        0xac0c, 0xffffffff, 0x32761054,
 443        0x88d4, 0x0000001f, 0x00000010,
 444        0x15c0, 0x000c0fc0, 0x000c0400
 445};
 446
 447static const u32 verde_golden_rlc_registers[] =
 448{
 449        0xc424, 0xffffffff, 0x033f1005,
 450        0xc47c, 0xffffffff, 0x10808020,
 451        0xc488, 0xffffffff, 0x00800008,
 452        0xc314, 0xffffffff, 0x00001000,
 453        0xc30c, 0xffffffff, 0x80010014
 454};
 455
 456static const u32 verde_golden_registers[] =
 457{
 458        0x9a10, 0x00010000, 0x00018208,
 459        0x9830, 0xffffffff, 0x00000000,
 460        0x9834, 0xf00fffff, 0x00000400,
 461        0x9838, 0x0002021c, 0x00020200,
 462        0xc78, 0x00000080, 0x00000000,
 463        0xd030, 0x000300c0, 0x00800040,
 464        0xd030, 0x000300c0, 0x00800040,
 465        0xd830, 0x000300c0, 0x00800040,
 466        0xd830, 0x000300c0, 0x00800040,
 467        0x5bb0, 0x000000f0, 0x00000070,
 468        0x5bc0, 0x00200000, 0x50100000,
 469        0x7030, 0x31000311, 0x00000011,
 470        0x2ae4, 0x00073ffe, 0x000022a2,
 471        0x2ae4, 0x00073ffe, 0x000022a2,
 472        0x2ae4, 0x00073ffe, 0x000022a2,
 473        0x240c, 0x000007ff, 0x00000000,
 474        0x240c, 0x000007ff, 0x00000000,
 475        0x240c, 0x000007ff, 0x00000000,
 476        0x8a14, 0xf000001f, 0x00000007,
 477        0x8a14, 0xf000001f, 0x00000007,
 478        0x8a14, 0xf000001f, 0x00000007,
 479        0x8b24, 0xffffffff, 0x00ffffff,
 480        0x8b10, 0x0000ff0f, 0x00000000,
 481        0x28a4c, 0x07ffffff, 0x4e000000,
 482        0x28350, 0x3f3f3fff, 0x0000124a,
 483        0x28350, 0x3f3f3fff, 0x0000124a,
 484        0x28350, 0x3f3f3fff, 0x0000124a,
 485        0x30, 0x000000ff, 0x0040,
 486        0x34, 0x00000040, 0x00004040,
 487        0x9100, 0x07ffffff, 0x03000000,
 488        0x9100, 0x07ffffff, 0x03000000,
 489        0x8e88, 0x01ff1f3f, 0x00000000,
 490        0x8e88, 0x01ff1f3f, 0x00000000,
 491        0x8e88, 0x01ff1f3f, 0x00000000,
 492        0x8e84, 0x01ff1f3f, 0x00000000,
 493        0x8e84, 0x01ff1f3f, 0x00000000,
 494        0x8e84, 0x01ff1f3f, 0x00000000,
 495        0x9060, 0x0000007f, 0x00000020,
 496        0x9508, 0x00010000, 0x00010000,
 497        0xac14, 0x000003ff, 0x00000003,
 498        0xac14, 0x000003ff, 0x00000003,
 499        0xac14, 0x000003ff, 0x00000003,
 500        0xac10, 0xffffffff, 0x00000000,
 501        0xac10, 0xffffffff, 0x00000000,
 502        0xac10, 0xffffffff, 0x00000000,
 503        0xac0c, 0xffffffff, 0x00001032,
 504        0xac0c, 0xffffffff, 0x00001032,
 505        0xac0c, 0xffffffff, 0x00001032,
 506        0x88d4, 0x0000001f, 0x00000010,
 507        0x88d4, 0x0000001f, 0x00000010,
 508        0x88d4, 0x0000001f, 0x00000010,
 509        0x15c0, 0x000c0fc0, 0x000c0400
 510};
 511
 512static const u32 oland_golden_rlc_registers[] =
 513{
 514        0xc424, 0xffffffff, 0x00601005,
 515        0xc47c, 0xffffffff, 0x10104040,
 516        0xc488, 0xffffffff, 0x0100000a,
 517        0xc314, 0xffffffff, 0x00000800,
 518        0xc30c, 0xffffffff, 0x800000f4
 519};
 520
 521static const u32 oland_golden_registers[] =
 522{
 523        0x9a10, 0x00010000, 0x00018208,
 524        0x9830, 0xffffffff, 0x00000000,
 525        0x9834, 0xf00fffff, 0x00000400,
 526        0x9838, 0x0002021c, 0x00020200,
 527        0xc78, 0x00000080, 0x00000000,
 528        0xd030, 0x000300c0, 0x00800040,
 529        0xd830, 0x000300c0, 0x00800040,
 530        0x5bb0, 0x000000f0, 0x00000070,
 531        0x5bc0, 0x00200000, 0x50100000,
 532        0x7030, 0x31000311, 0x00000011,
 533        0x2ae4, 0x00073ffe, 0x000022a2,
 534        0x240c, 0x000007ff, 0x00000000,
 535        0x8a14, 0xf000001f, 0x00000007,
 536        0x8b24, 0xffffffff, 0x00ffffff,
 537        0x8b10, 0x0000ff0f, 0x00000000,
 538        0x28a4c, 0x07ffffff, 0x4e000000,
 539        0x28350, 0x3f3f3fff, 0x00000082,
 540        0x30, 0x000000ff, 0x0040,
 541        0x34, 0x00000040, 0x00004040,
 542        0x9100, 0x07ffffff, 0x03000000,
 543        0x9060, 0x0000007f, 0x00000020,
 544        0x9508, 0x00010000, 0x00010000,
 545        0xac14, 0x000003ff, 0x000000f3,
 546        0xac10, 0xffffffff, 0x00000000,
 547        0xac0c, 0xffffffff, 0x00003210,
 548        0x88d4, 0x0000001f, 0x00000010,
 549        0x15c0, 0x000c0fc0, 0x000c0400
 550};
 551
 552static const u32 hainan_golden_registers[] =
 553{
 554        0x9a10, 0x00010000, 0x00018208,
 555        0x9830, 0xffffffff, 0x00000000,
 556        0x9834, 0xf00fffff, 0x00000400,
 557        0x9838, 0x0002021c, 0x00020200,
 558        0xd0c0, 0xff000fff, 0x00000100,
 559        0xd030, 0x000300c0, 0x00800040,
 560        0xd8c0, 0xff000fff, 0x00000100,
 561        0xd830, 0x000300c0, 0x00800040,
 562        0x2ae4, 0x00073ffe, 0x000022a2,
 563        0x240c, 0x000007ff, 0x00000000,
 564        0x8a14, 0xf000001f, 0x00000007,
 565        0x8b24, 0xffffffff, 0x00ffffff,
 566        0x8b10, 0x0000ff0f, 0x00000000,
 567        0x28a4c, 0x07ffffff, 0x4e000000,
 568        0x28350, 0x3f3f3fff, 0x00000000,
 569        0x30, 0x000000ff, 0x0040,
 570        0x34, 0x00000040, 0x00004040,
 571        0x9100, 0x03e00000, 0x03600000,
 572        0x9060, 0x0000007f, 0x00000020,
 573        0x9508, 0x00010000, 0x00010000,
 574        0xac14, 0x000003ff, 0x000000f1,
 575        0xac10, 0xffffffff, 0x00000000,
 576        0xac0c, 0xffffffff, 0x00003210,
 577        0x88d4, 0x0000001f, 0x00000010,
 578        0x15c0, 0x000c0fc0, 0x000c0400
 579};
 580
 581static const u32 hainan_golden_registers2[] =
 582{
 583        0x98f8, 0xffffffff, 0x02010001
 584};
 585
 586static const u32 tahiti_mgcg_cgcg_init[] =
 587{
 588        0xc400, 0xffffffff, 0xfffffffc,
 589        0x802c, 0xffffffff, 0xe0000000,
 590        0x9a60, 0xffffffff, 0x00000100,
 591        0x92a4, 0xffffffff, 0x00000100,
 592        0xc164, 0xffffffff, 0x00000100,
 593        0x9774, 0xffffffff, 0x00000100,
 594        0x8984, 0xffffffff, 0x06000100,
 595        0x8a18, 0xffffffff, 0x00000100,
 596        0x92a0, 0xffffffff, 0x00000100,
 597        0xc380, 0xffffffff, 0x00000100,
 598        0x8b28, 0xffffffff, 0x00000100,
 599        0x9144, 0xffffffff, 0x00000100,
 600        0x8d88, 0xffffffff, 0x00000100,
 601        0x8d8c, 0xffffffff, 0x00000100,
 602        0x9030, 0xffffffff, 0x00000100,
 603        0x9034, 0xffffffff, 0x00000100,
 604        0x9038, 0xffffffff, 0x00000100,
 605        0x903c, 0xffffffff, 0x00000100,
 606        0xad80, 0xffffffff, 0x00000100,
 607        0xac54, 0xffffffff, 0x00000100,
 608        0x897c, 0xffffffff, 0x06000100,
 609        0x9868, 0xffffffff, 0x00000100,
 610        0x9510, 0xffffffff, 0x00000100,
 611        0xaf04, 0xffffffff, 0x00000100,
 612        0xae04, 0xffffffff, 0x00000100,
 613        0x949c, 0xffffffff, 0x00000100,
 614        0x802c, 0xffffffff, 0xe0000000,
 615        0x9160, 0xffffffff, 0x00010000,
 616        0x9164, 0xffffffff, 0x00030002,
 617        0x9168, 0xffffffff, 0x00040007,
 618        0x916c, 0xffffffff, 0x00060005,
 619        0x9170, 0xffffffff, 0x00090008,
 620        0x9174, 0xffffffff, 0x00020001,
 621        0x9178, 0xffffffff, 0x00040003,
 622        0x917c, 0xffffffff, 0x00000007,
 623        0x9180, 0xffffffff, 0x00060005,
 624        0x9184, 0xffffffff, 0x00090008,
 625        0x9188, 0xffffffff, 0x00030002,
 626        0x918c, 0xffffffff, 0x00050004,
 627        0x9190, 0xffffffff, 0x00000008,
 628        0x9194, 0xffffffff, 0x00070006,
 629        0x9198, 0xffffffff, 0x000a0009,
 630        0x919c, 0xffffffff, 0x00040003,
 631        0x91a0, 0xffffffff, 0x00060005,
 632        0x91a4, 0xffffffff, 0x00000009,
 633        0x91a8, 0xffffffff, 0x00080007,
 634        0x91ac, 0xffffffff, 0x000b000a,
 635        0x91b0, 0xffffffff, 0x00050004,
 636        0x91b4, 0xffffffff, 0x00070006,
 637        0x91b8, 0xffffffff, 0x0008000b,
 638        0x91bc, 0xffffffff, 0x000a0009,
 639        0x91c0, 0xffffffff, 0x000d000c,
 640        0x91c4, 0xffffffff, 0x00060005,
 641        0x91c8, 0xffffffff, 0x00080007,
 642        0x91cc, 0xffffffff, 0x0000000b,
 643        0x91d0, 0xffffffff, 0x000a0009,
 644        0x91d4, 0xffffffff, 0x000d000c,
 645        0x91d8, 0xffffffff, 0x00070006,
 646        0x91dc, 0xffffffff, 0x00090008,
 647        0x91e0, 0xffffffff, 0x0000000c,
 648        0x91e4, 0xffffffff, 0x000b000a,
 649        0x91e8, 0xffffffff, 0x000e000d,
 650        0x91ec, 0xffffffff, 0x00080007,
 651        0x91f0, 0xffffffff, 0x000a0009,
 652        0x91f4, 0xffffffff, 0x0000000d,
 653        0x91f8, 0xffffffff, 0x000c000b,
 654        0x91fc, 0xffffffff, 0x000f000e,
 655        0x9200, 0xffffffff, 0x00090008,
 656        0x9204, 0xffffffff, 0x000b000a,
 657        0x9208, 0xffffffff, 0x000c000f,
 658        0x920c, 0xffffffff, 0x000e000d,
 659        0x9210, 0xffffffff, 0x00110010,
 660        0x9214, 0xffffffff, 0x000a0009,
 661        0x9218, 0xffffffff, 0x000c000b,
 662        0x921c, 0xffffffff, 0x0000000f,
 663        0x9220, 0xffffffff, 0x000e000d,
 664        0x9224, 0xffffffff, 0x00110010,
 665        0x9228, 0xffffffff, 0x000b000a,
 666        0x922c, 0xffffffff, 0x000d000c,
 667        0x9230, 0xffffffff, 0x00000010,
 668        0x9234, 0xffffffff, 0x000f000e,
 669        0x9238, 0xffffffff, 0x00120011,
 670        0x923c, 0xffffffff, 0x000c000b,
 671        0x9240, 0xffffffff, 0x000e000d,
 672        0x9244, 0xffffffff, 0x00000011,
 673        0x9248, 0xffffffff, 0x0010000f,
 674        0x924c, 0xffffffff, 0x00130012,
 675        0x9250, 0xffffffff, 0x000d000c,
 676        0x9254, 0xffffffff, 0x000f000e,
 677        0x9258, 0xffffffff, 0x00100013,
 678        0x925c, 0xffffffff, 0x00120011,
 679        0x9260, 0xffffffff, 0x00150014,
 680        0x9264, 0xffffffff, 0x000e000d,
 681        0x9268, 0xffffffff, 0x0010000f,
 682        0x926c, 0xffffffff, 0x00000013,
 683        0x9270, 0xffffffff, 0x00120011,
 684        0x9274, 0xffffffff, 0x00150014,
 685        0x9278, 0xffffffff, 0x000f000e,
 686        0x927c, 0xffffffff, 0x00110010,
 687        0x9280, 0xffffffff, 0x00000014,
 688        0x9284, 0xffffffff, 0x00130012,
 689        0x9288, 0xffffffff, 0x00160015,
 690        0x928c, 0xffffffff, 0x0010000f,
 691        0x9290, 0xffffffff, 0x00120011,
 692        0x9294, 0xffffffff, 0x00000015,
 693        0x9298, 0xffffffff, 0x00140013,
 694        0x929c, 0xffffffff, 0x00170016,
 695        0x9150, 0xffffffff, 0x96940200,
 696        0x8708, 0xffffffff, 0x00900100,
 697        0xc478, 0xffffffff, 0x00000080,
 698        0xc404, 0xffffffff, 0x0020003f,
 699        0x30, 0xffffffff, 0x0000001c,
 700        0x34, 0x000f0000, 0x000f0000,
 701        0x160c, 0xffffffff, 0x00000100,
 702        0x1024, 0xffffffff, 0x00000100,
 703        0x102c, 0x00000101, 0x00000000,
 704        0x20a8, 0xffffffff, 0x00000104,
 705        0x264c, 0x000c0000, 0x000c0000,
 706        0x2648, 0x000c0000, 0x000c0000,
 707        0x55e4, 0xff000fff, 0x00000100,
 708        0x55e8, 0x00000001, 0x00000001,
 709        0x2f50, 0x00000001, 0x00000001,
 710        0x30cc, 0xc0000fff, 0x00000104,
 711        0xc1e4, 0x00000001, 0x00000001,
 712        0xd0c0, 0xfffffff0, 0x00000100,
 713        0xd8c0, 0xfffffff0, 0x00000100
 714};
 715
 716static const u32 pitcairn_mgcg_cgcg_init[] =
 717{
 718        0xc400, 0xffffffff, 0xfffffffc,
 719        0x802c, 0xffffffff, 0xe0000000,
 720        0x9a60, 0xffffffff, 0x00000100,
 721        0x92a4, 0xffffffff, 0x00000100,
 722        0xc164, 0xffffffff, 0x00000100,
 723        0x9774, 0xffffffff, 0x00000100,
 724        0x8984, 0xffffffff, 0x06000100,
 725        0x8a18, 0xffffffff, 0x00000100,
 726        0x92a0, 0xffffffff, 0x00000100,
 727        0xc380, 0xffffffff, 0x00000100,
 728        0x8b28, 0xffffffff, 0x00000100,
 729        0x9144, 0xffffffff, 0x00000100,
 730        0x8d88, 0xffffffff, 0x00000100,
 731        0x8d8c, 0xffffffff, 0x00000100,
 732        0x9030, 0xffffffff, 0x00000100,
 733        0x9034, 0xffffffff, 0x00000100,
 734        0x9038, 0xffffffff, 0x00000100,
 735        0x903c, 0xffffffff, 0x00000100,
 736        0xad80, 0xffffffff, 0x00000100,
 737        0xac54, 0xffffffff, 0x00000100,
 738        0x897c, 0xffffffff, 0x06000100,
 739        0x9868, 0xffffffff, 0x00000100,
 740        0x9510, 0xffffffff, 0x00000100,
 741        0xaf04, 0xffffffff, 0x00000100,
 742        0xae04, 0xffffffff, 0x00000100,
 743        0x949c, 0xffffffff, 0x00000100,
 744        0x802c, 0xffffffff, 0xe0000000,
 745        0x9160, 0xffffffff, 0x00010000,
 746        0x9164, 0xffffffff, 0x00030002,
 747        0x9168, 0xffffffff, 0x00040007,
 748        0x916c, 0xffffffff, 0x00060005,
 749        0x9170, 0xffffffff, 0x00090008,
 750        0x9174, 0xffffffff, 0x00020001,
 751        0x9178, 0xffffffff, 0x00040003,
 752        0x917c, 0xffffffff, 0x00000007,
 753        0x9180, 0xffffffff, 0x00060005,
 754        0x9184, 0xffffffff, 0x00090008,
 755        0x9188, 0xffffffff, 0x00030002,
 756        0x918c, 0xffffffff, 0x00050004,
 757        0x9190, 0xffffffff, 0x00000008,
 758        0x9194, 0xffffffff, 0x00070006,
 759        0x9198, 0xffffffff, 0x000a0009,
 760        0x919c, 0xffffffff, 0x00040003,
 761        0x91a0, 0xffffffff, 0x00060005,
 762        0x91a4, 0xffffffff, 0x00000009,
 763        0x91a8, 0xffffffff, 0x00080007,
 764        0x91ac, 0xffffffff, 0x000b000a,
 765        0x91b0, 0xffffffff, 0x00050004,
 766        0x91b4, 0xffffffff, 0x00070006,
 767        0x91b8, 0xffffffff, 0x0008000b,
 768        0x91bc, 0xffffffff, 0x000a0009,
 769        0x91c0, 0xffffffff, 0x000d000c,
 770        0x9200, 0xffffffff, 0x00090008,
 771        0x9204, 0xffffffff, 0x000b000a,
 772        0x9208, 0xffffffff, 0x000c000f,
 773        0x920c, 0xffffffff, 0x000e000d,
 774        0x9210, 0xffffffff, 0x00110010,
 775        0x9214, 0xffffffff, 0x000a0009,
 776        0x9218, 0xffffffff, 0x000c000b,
 777        0x921c, 0xffffffff, 0x0000000f,
 778        0x9220, 0xffffffff, 0x000e000d,
 779        0x9224, 0xffffffff, 0x00110010,
 780        0x9228, 0xffffffff, 0x000b000a,
 781        0x922c, 0xffffffff, 0x000d000c,
 782        0x9230, 0xffffffff, 0x00000010,
 783        0x9234, 0xffffffff, 0x000f000e,
 784        0x9238, 0xffffffff, 0x00120011,
 785        0x923c, 0xffffffff, 0x000c000b,
 786        0x9240, 0xffffffff, 0x000e000d,
 787        0x9244, 0xffffffff, 0x00000011,
 788        0x9248, 0xffffffff, 0x0010000f,
 789        0x924c, 0xffffffff, 0x00130012,
 790        0x9250, 0xffffffff, 0x000d000c,
 791        0x9254, 0xffffffff, 0x000f000e,
 792        0x9258, 0xffffffff, 0x00100013,
 793        0x925c, 0xffffffff, 0x00120011,
 794        0x9260, 0xffffffff, 0x00150014,
 795        0x9150, 0xffffffff, 0x96940200,
 796        0x8708, 0xffffffff, 0x00900100,
 797        0xc478, 0xffffffff, 0x00000080,
 798        0xc404, 0xffffffff, 0x0020003f,
 799        0x30, 0xffffffff, 0x0000001c,
 800        0x34, 0x000f0000, 0x000f0000,
 801        0x160c, 0xffffffff, 0x00000100,
 802        0x1024, 0xffffffff, 0x00000100,
 803        0x102c, 0x00000101, 0x00000000,
 804        0x20a8, 0xffffffff, 0x00000104,
 805        0x55e4, 0xff000fff, 0x00000100,
 806        0x55e8, 0x00000001, 0x00000001,
 807        0x2f50, 0x00000001, 0x00000001,
 808        0x30cc, 0xc0000fff, 0x00000104,
 809        0xc1e4, 0x00000001, 0x00000001,
 810        0xd0c0, 0xfffffff0, 0x00000100,
 811        0xd8c0, 0xfffffff0, 0x00000100
 812};
 813
 814static const u32 verde_mgcg_cgcg_init[] =
 815{
 816        0xc400, 0xffffffff, 0xfffffffc,
 817        0x802c, 0xffffffff, 0xe0000000,
 818        0x9a60, 0xffffffff, 0x00000100,
 819        0x92a4, 0xffffffff, 0x00000100,
 820        0xc164, 0xffffffff, 0x00000100,
 821        0x9774, 0xffffffff, 0x00000100,
 822        0x8984, 0xffffffff, 0x06000100,
 823        0x8a18, 0xffffffff, 0x00000100,
 824        0x92a0, 0xffffffff, 0x00000100,
 825        0xc380, 0xffffffff, 0x00000100,
 826        0x8b28, 0xffffffff, 0x00000100,
 827        0x9144, 0xffffffff, 0x00000100,
 828        0x8d88, 0xffffffff, 0x00000100,
 829        0x8d8c, 0xffffffff, 0x00000100,
 830        0x9030, 0xffffffff, 0x00000100,
 831        0x9034, 0xffffffff, 0x00000100,
 832        0x9038, 0xffffffff, 0x00000100,
 833        0x903c, 0xffffffff, 0x00000100,
 834        0xad80, 0xffffffff, 0x00000100,
 835        0xac54, 0xffffffff, 0x00000100,
 836        0x897c, 0xffffffff, 0x06000100,
 837        0x9868, 0xffffffff, 0x00000100,
 838        0x9510, 0xffffffff, 0x00000100,
 839        0xaf04, 0xffffffff, 0x00000100,
 840        0xae04, 0xffffffff, 0x00000100,
 841        0x949c, 0xffffffff, 0x00000100,
 842        0x802c, 0xffffffff, 0xe0000000,
 843        0x9160, 0xffffffff, 0x00010000,
 844        0x9164, 0xffffffff, 0x00030002,
 845        0x9168, 0xffffffff, 0x00040007,
 846        0x916c, 0xffffffff, 0x00060005,
 847        0x9170, 0xffffffff, 0x00090008,
 848        0x9174, 0xffffffff, 0x00020001,
 849        0x9178, 0xffffffff, 0x00040003,
 850        0x917c, 0xffffffff, 0x00000007,
 851        0x9180, 0xffffffff, 0x00060005,
 852        0x9184, 0xffffffff, 0x00090008,
 853        0x9188, 0xffffffff, 0x00030002,
 854        0x918c, 0xffffffff, 0x00050004,
 855        0x9190, 0xffffffff, 0x00000008,
 856        0x9194, 0xffffffff, 0x00070006,
 857        0x9198, 0xffffffff, 0x000a0009,
 858        0x919c, 0xffffffff, 0x00040003,
 859        0x91a0, 0xffffffff, 0x00060005,
 860        0x91a4, 0xffffffff, 0x00000009,
 861        0x91a8, 0xffffffff, 0x00080007,
 862        0x91ac, 0xffffffff, 0x000b000a,
 863        0x91b0, 0xffffffff, 0x00050004,
 864        0x91b4, 0xffffffff, 0x00070006,
 865        0x91b8, 0xffffffff, 0x0008000b,
 866        0x91bc, 0xffffffff, 0x000a0009,
 867        0x91c0, 0xffffffff, 0x000d000c,
 868        0x9200, 0xffffffff, 0x00090008,
 869        0x9204, 0xffffffff, 0x000b000a,
 870        0x9208, 0xffffffff, 0x000c000f,
 871        0x920c, 0xffffffff, 0x000e000d,
 872        0x9210, 0xffffffff, 0x00110010,
 873        0x9214, 0xffffffff, 0x000a0009,
 874        0x9218, 0xffffffff, 0x000c000b,
 875        0x921c, 0xffffffff, 0x0000000f,
 876        0x9220, 0xffffffff, 0x000e000d,
 877        0x9224, 0xffffffff, 0x00110010,
 878        0x9228, 0xffffffff, 0x000b000a,
 879        0x922c, 0xffffffff, 0x000d000c,
 880        0x9230, 0xffffffff, 0x00000010,
 881        0x9234, 0xffffffff, 0x000f000e,
 882        0x9238, 0xffffffff, 0x00120011,
 883        0x923c, 0xffffffff, 0x000c000b,
 884        0x9240, 0xffffffff, 0x000e000d,
 885        0x9244, 0xffffffff, 0x00000011,
 886        0x9248, 0xffffffff, 0x0010000f,
 887        0x924c, 0xffffffff, 0x00130012,
 888        0x9250, 0xffffffff, 0x000d000c,
 889        0x9254, 0xffffffff, 0x000f000e,
 890        0x9258, 0xffffffff, 0x00100013,
 891        0x925c, 0xffffffff, 0x00120011,
 892        0x9260, 0xffffffff, 0x00150014,
 893        0x9150, 0xffffffff, 0x96940200,
 894        0x8708, 0xffffffff, 0x00900100,
 895        0xc478, 0xffffffff, 0x00000080,
 896        0xc404, 0xffffffff, 0x0020003f,
 897        0x30, 0xffffffff, 0x0000001c,
 898        0x34, 0x000f0000, 0x000f0000,
 899        0x160c, 0xffffffff, 0x00000100,
 900        0x1024, 0xffffffff, 0x00000100,
 901        0x102c, 0x00000101, 0x00000000,
 902        0x20a8, 0xffffffff, 0x00000104,
 903        0x264c, 0x000c0000, 0x000c0000,
 904        0x2648, 0x000c0000, 0x000c0000,
 905        0x55e4, 0xff000fff, 0x00000100,
 906        0x55e8, 0x00000001, 0x00000001,
 907        0x2f50, 0x00000001, 0x00000001,
 908        0x30cc, 0xc0000fff, 0x00000104,
 909        0xc1e4, 0x00000001, 0x00000001,
 910        0xd0c0, 0xfffffff0, 0x00000100,
 911        0xd8c0, 0xfffffff0, 0x00000100
 912};
 913
 914static const u32 oland_mgcg_cgcg_init[] =
 915{
 916        0xc400, 0xffffffff, 0xfffffffc,
 917        0x802c, 0xffffffff, 0xe0000000,
 918        0x9a60, 0xffffffff, 0x00000100,
 919        0x92a4, 0xffffffff, 0x00000100,
 920        0xc164, 0xffffffff, 0x00000100,
 921        0x9774, 0xffffffff, 0x00000100,
 922        0x8984, 0xffffffff, 0x06000100,
 923        0x8a18, 0xffffffff, 0x00000100,
 924        0x92a0, 0xffffffff, 0x00000100,
 925        0xc380, 0xffffffff, 0x00000100,
 926        0x8b28, 0xffffffff, 0x00000100,
 927        0x9144, 0xffffffff, 0x00000100,
 928        0x8d88, 0xffffffff, 0x00000100,
 929        0x8d8c, 0xffffffff, 0x00000100,
 930        0x9030, 0xffffffff, 0x00000100,
 931        0x9034, 0xffffffff, 0x00000100,
 932        0x9038, 0xffffffff, 0x00000100,
 933        0x903c, 0xffffffff, 0x00000100,
 934        0xad80, 0xffffffff, 0x00000100,
 935        0xac54, 0xffffffff, 0x00000100,
 936        0x897c, 0xffffffff, 0x06000100,
 937        0x9868, 0xffffffff, 0x00000100,
 938        0x9510, 0xffffffff, 0x00000100,
 939        0xaf04, 0xffffffff, 0x00000100,
 940        0xae04, 0xffffffff, 0x00000100,
 941        0x949c, 0xffffffff, 0x00000100,
 942        0x802c, 0xffffffff, 0xe0000000,
 943        0x9160, 0xffffffff, 0x00010000,
 944        0x9164, 0xffffffff, 0x00030002,
 945        0x9168, 0xffffffff, 0x00040007,
 946        0x916c, 0xffffffff, 0x00060005,
 947        0x9170, 0xffffffff, 0x00090008,
 948        0x9174, 0xffffffff, 0x00020001,
 949        0x9178, 0xffffffff, 0x00040003,
 950        0x917c, 0xffffffff, 0x00000007,
 951        0x9180, 0xffffffff, 0x00060005,
 952        0x9184, 0xffffffff, 0x00090008,
 953        0x9188, 0xffffffff, 0x00030002,
 954        0x918c, 0xffffffff, 0x00050004,
 955        0x9190, 0xffffffff, 0x00000008,
 956        0x9194, 0xffffffff, 0x00070006,
 957        0x9198, 0xffffffff, 0x000a0009,
 958        0x919c, 0xffffffff, 0x00040003,
 959        0x91a0, 0xffffffff, 0x00060005,
 960        0x91a4, 0xffffffff, 0x00000009,
 961        0x91a8, 0xffffffff, 0x00080007,
 962        0x91ac, 0xffffffff, 0x000b000a,
 963        0x91b0, 0xffffffff, 0x00050004,
 964        0x91b4, 0xffffffff, 0x00070006,
 965        0x91b8, 0xffffffff, 0x0008000b,
 966        0x91bc, 0xffffffff, 0x000a0009,
 967        0x91c0, 0xffffffff, 0x000d000c,
 968        0x91c4, 0xffffffff, 0x00060005,
 969        0x91c8, 0xffffffff, 0x00080007,
 970        0x91cc, 0xffffffff, 0x0000000b,
 971        0x91d0, 0xffffffff, 0x000a0009,
 972        0x91d4, 0xffffffff, 0x000d000c,
 973        0x9150, 0xffffffff, 0x96940200,
 974        0x8708, 0xffffffff, 0x00900100,
 975        0xc478, 0xffffffff, 0x00000080,
 976        0xc404, 0xffffffff, 0x0020003f,
 977        0x30, 0xffffffff, 0x0000001c,
 978        0x34, 0x000f0000, 0x000f0000,
 979        0x160c, 0xffffffff, 0x00000100,
 980        0x1024, 0xffffffff, 0x00000100,
 981        0x102c, 0x00000101, 0x00000000,
 982        0x20a8, 0xffffffff, 0x00000104,
 983        0x264c, 0x000c0000, 0x000c0000,
 984        0x2648, 0x000c0000, 0x000c0000,
 985        0x55e4, 0xff000fff, 0x00000100,
 986        0x55e8, 0x00000001, 0x00000001,
 987        0x2f50, 0x00000001, 0x00000001,
 988        0x30cc, 0xc0000fff, 0x00000104,
 989        0xc1e4, 0x00000001, 0x00000001,
 990        0xd0c0, 0xfffffff0, 0x00000100,
 991        0xd8c0, 0xfffffff0, 0x00000100
 992};
 993
 994static const u32 hainan_mgcg_cgcg_init[] =
 995{
 996        0xc400, 0xffffffff, 0xfffffffc,
 997        0x802c, 0xffffffff, 0xe0000000,
 998        0x9a60, 0xffffffff, 0x00000100,
 999        0x92a4, 0xffffffff, 0x00000100,
1000        0xc164, 0xffffffff, 0x00000100,

1001        0x9774, 0xffffffff, 0x00000100,
1002        0x8984, 0xffffffff, 0x06000100,
1003        0x8a18, 0xffffffff, 0x00000100,
1004        0x92a0, 0xffffffff, 0x00000100,
1005        0xc380, 0xffffffff, 0x00000100,
1006        0x8b28, 0xffffffff, 0x00000100,
1007        0x9144, 0xffffffff, 0x00000100,
1008        0x8d88, 0xffffffff, 0x00000100,
1009        0x8d8c, 0xffffffff, 0x00000100,
1010        0x9030, 0xffffffff, 0x00000100,
1011        0x9034, 0xffffffff, 0x00000100,
1012        0x9038, 0xffffffff, 0x00000100,
1013        0x903c, 0xffffffff, 0x00000100,
1014        0xad80, 0xffffffff, 0x00000100,
1015        0xac54, 0xffffffff, 0x00000100,
1016        0x897c, 0xffffffff, 0x06000100,
1017        0x9868, 0xffffffff, 0x00000100,
1018        0x9510, 0xffffffff, 0x00000100,
1019        0xaf04, 0xffffffff, 0x00000100,
1020        0xae04, 0xffffffff, 0x00000100,
1021        0x949c, 0xffffffff, 0x00000100,
1022        0x802c, 0xffffffff, 0xe0000000,
1023        0x9160, 0xffffffff, 0x00010000,
1024        0x9164, 0xffffffff, 0x00030002,
1025        0x9168, 0xffffffff, 0x00040007,
1026        0x916c, 0xffffffff, 0x00060005,
1027        0x9170, 0xffffffff, 0x00090008,
1028        0x9174, 0xffffffff, 0x00020001,
1029        0x9178, 0xffffffff, 0x00040003,
1030        0x917c, 0xffffffff, 0x00000007,
1031        0x9180, 0xffffffff, 0x00060005,
1032        0x9184, 0xffffffff, 0x00090008,
1033        0x9188, 0xffffffff, 0x00030002,
1034        0x918c, 0xffffffff, 0x00050004,
1035        0x9190, 0xffffffff, 0x00000008,
1036        0x9194, 0xffffffff, 0x00070006,
1037        0x9198, 0xffffffff, 0x000a0009,
1038        0x919c, 0xffffffff, 0x00040003,
1039        0x91a0, 0xffffffff, 0x00060005,
1040        0x91a4, 0xffffffff, 0x00000009,
1041        0x91a8, 0xffffffff, 0x00080007,
1042        0x91ac, 0xffffffff, 0x000b000a,
1043        0x91b0, 0xffffffff, 0x00050004,
1044        0x91b4, 0xffffffff, 0x00070006,
1045        0x91b8, 0xffffffff, 0x0008000b,
1046        0x91bc, 0xffffffff, 0x000a0009,
1047        0x91c0, 0xffffffff, 0x000d000c,
1048        0x91c4, 0xffffffff, 0x00060005,
1049        0x91c8, 0xffffffff, 0x00080007,
1050        0x91cc, 0xffffffff, 0x0000000b,
1051        0x91d0, 0xffffffff, 0x000a0009,
1052        0x91d4, 0xffffffff, 0x000d000c,
1053        0x9150, 0xffffffff, 0x96940200,
1054        0x8708, 0xffffffff, 0x00900100,
1055        0xc478, 0xffffffff, 0x00000080,
1056        0xc404, 0xffffffff, 0x0020003f,
1057        0x30, 0xffffffff, 0x0000001c,
1058        0x34, 0x000f0000, 0x000f0000,
1059        0x160c, 0xffffffff, 0x00000100,
1060        0x1024, 0xffffffff, 0x00000100,
1061        0x20a8, 0xffffffff, 0x00000104,
1062        0x264c, 0x000c0000, 0x000c0000,
1063        0x2648, 0x000c0000, 0x000c0000,
1064        0x2f50, 0x00000001, 0x00000001,
1065        0x30cc, 0xc0000fff, 0x00000104,
1066        0xc1e4, 0x00000001, 0x00000001,
1067        0xd0c0, 0xfffffff0, 0x00000100,
1068        0xd8c0, 0xfffffff0, 0x00000100
1069};
1070
1071static u32 verde_pg_init[] =
1072{
1073        0x353c, 0xffffffff, 0x40000,
1074        0x3538, 0xffffffff, 0x200010ff,
1075        0x353c, 0xffffffff, 0x0,
1076        0x353c, 0xffffffff, 0x0,
1077        0x353c, 0xffffffff, 0x0,
1078        0x353c, 0xffffffff, 0x0,
1079        0x353c, 0xffffffff, 0x0,
1080        0x353c, 0xffffffff, 0x7007,
1081        0x3538, 0xffffffff, 0x300010ff,
1082        0x353c, 0xffffffff, 0x0,
1083        0x353c, 0xffffffff, 0x0,
1084        0x353c, 0xffffffff, 0x0,
1085        0x353c, 0xffffffff, 0x0,
1086        0x353c, 0xffffffff, 0x0,
1087        0x353c, 0xffffffff, 0x400000,
1088        0x3538, 0xffffffff, 0x100010ff,
1089        0x353c, 0xffffffff, 0x0,
1090        0x353c, 0xffffffff, 0x0,
1091        0x353c, 0xffffffff, 0x0,
1092        0x353c, 0xffffffff, 0x0,
1093        0x353c, 0xffffffff, 0x0,
1094        0x353c, 0xffffffff, 0x120200,
1095        0x3538, 0xffffffff, 0x500010ff,
1096        0x353c, 0xffffffff, 0x0,
1097        0x353c, 0xffffffff, 0x0,
1098        0x353c, 0xffffffff, 0x0,
1099        0x353c, 0xffffffff, 0x0,
1100        0x353c, 0xffffffff, 0x0,
1101        0x353c, 0xffffffff, 0x1e1e16,
1102        0x3538, 0xffffffff, 0x600010ff,
1103        0x353c, 0xffffffff, 0x0,
1104        0x353c, 0xffffffff, 0x0,
1105        0x353c, 0xffffffff, 0x0,
1106        0x353c, 0xffffffff, 0x0,
1107        0x353c, 0xffffffff, 0x0,
1108        0x353c, 0xffffffff, 0x171f1e,
1109        0x3538, 0xffffffff, 0x700010ff,
1110        0x353c, 0xffffffff, 0x0,
1111        0x353c, 0xffffffff, 0x0,
1112        0x353c, 0xffffffff, 0x0,
1113        0x353c, 0xffffffff, 0x0,
1114        0x353c, 0xffffffff, 0x0,
1115        0x353c, 0xffffffff, 0x0,
1116        0x3538, 0xffffffff, 0x9ff,
1117        0x3500, 0xffffffff, 0x0,
1118        0x3504, 0xffffffff, 0x10000800,
1119        0x3504, 0xffffffff, 0xf,
1120        0x3504, 0xffffffff, 0xf,
1121        0x3500, 0xffffffff, 0x4,
1122        0x3504, 0xffffffff, 0x1000051e,
1123        0x3504, 0xffffffff, 0xffff,
1124        0x3504, 0xffffffff, 0xffff,
1125        0x3500, 0xffffffff, 0x8,
1126        0x3504, 0xffffffff, 0x80500,
1127        0x3500, 0xffffffff, 0x12,
1128        0x3504, 0xffffffff, 0x9050c,
1129        0x3500, 0xffffffff, 0x1d,
1130        0x3504, 0xffffffff, 0xb052c,
1131        0x3500, 0xffffffff, 0x2a,
1132        0x3504, 0xffffffff, 0x1053e,
1133        0x3500, 0xffffffff, 0x2d,
1134        0x3504, 0xffffffff, 0x10546,
1135        0x3500, 0xffffffff, 0x30,
1136        0x3504, 0xffffffff, 0xa054e,
1137        0x3500, 0xffffffff, 0x3c,
1138        0x3504, 0xffffffff, 0x1055f,
1139        0x3500, 0xffffffff, 0x3f,
1140        0x3504, 0xffffffff, 0x10567,
1141        0x3500, 0xffffffff, 0x42,
1142        0x3504, 0xffffffff, 0x1056f,
1143        0x3500, 0xffffffff, 0x45,
1144        0x3504, 0xffffffff, 0x10572,
1145        0x3500, 0xffffffff, 0x48,
1146        0x3504, 0xffffffff, 0x20575,
1147        0x3500, 0xffffffff, 0x4c,
1148        0x3504, 0xffffffff, 0x190801,
1149        0x3500, 0xffffffff, 0x67,
1150        0x3504, 0xffffffff, 0x1082a,
1151        0x3500, 0xffffffff, 0x6a,
1152        0x3504, 0xffffffff, 0x1b082d,
1153        0x3500, 0xffffffff, 0x87,
1154        0x3504, 0xffffffff, 0x310851,
1155        0x3500, 0xffffffff, 0xba,
1156        0x3504, 0xffffffff, 0x891,
1157        0x3500, 0xffffffff, 0xbc,
1158        0x3504, 0xffffffff, 0x893,
1159        0x3500, 0xffffffff, 0xbe,
1160        0x3504, 0xffffffff, 0x20895,
1161        0x3500, 0xffffffff, 0xc2,
1162        0x3504, 0xffffffff, 0x20899,
1163        0x3500, 0xffffffff, 0xc6,
1164        0x3504, 0xffffffff, 0x2089d,
1165        0x3500, 0xffffffff, 0xca,
1166        0x3504, 0xffffffff, 0x8a1,
1167        0x3500, 0xffffffff, 0xcc,
1168        0x3504, 0xffffffff, 0x8a3,
1169        0x3500, 0xffffffff, 0xce,
1170        0x3504, 0xffffffff, 0x308a5,
1171        0x3500, 0xffffffff, 0xd3,
1172        0x3504, 0xffffffff, 0x6d08cd,
1173        0x3500, 0xffffffff, 0x142,
1174        0x3504, 0xffffffff, 0x2000095a,
1175        0x3504, 0xffffffff, 0x1,
1176        0x3500, 0xffffffff, 0x144,
1177        0x3504, 0xffffffff, 0x301f095b,
1178        0x3500, 0xffffffff, 0x165,
1179        0x3504, 0xffffffff, 0xc094d,
1180        0x3500, 0xffffffff, 0x173,
1181        0x3504, 0xffffffff, 0xf096d,
1182        0x3500, 0xffffffff, 0x184,
1183        0x3504, 0xffffffff, 0x15097f,
1184        0x3500, 0xffffffff, 0x19b,
1185        0x3504, 0xffffffff, 0xc0998,
1186        0x3500, 0xffffffff, 0x1a9,
1187        0x3504, 0xffffffff, 0x409a7,
1188        0x3500, 0xffffffff, 0x1af,
1189        0x3504, 0xffffffff, 0xcdc,
1190        0x3500, 0xffffffff, 0x1b1,
1191        0x3504, 0xffffffff, 0x800,
1192        0x3508, 0xffffffff, 0x6c9b2000,
1193        0x3510, 0xfc00, 0x2000,
1194        0x3544, 0xffffffff, 0xfc0,
1195        0x28d4, 0x00000100, 0x100
1196};
1197
1198static void si_init_golden_registers(struct radeon_device *rdev)
1199{
1200        switch (rdev->family) {
1201        case CHIP_TAHITI:
1202                radeon_program_register_sequence(rdev,
1203                                                 tahiti_golden_registers,
1204                                                 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1205                radeon_program_register_sequence(rdev,
1206                                                 tahiti_golden_rlc_registers,
1207                                                 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1208                radeon_program_register_sequence(rdev,
1209                                                 tahiti_mgcg_cgcg_init,
1210                                                 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1211                radeon_program_register_sequence(rdev,
1212                                                 tahiti_golden_registers2,
1213                                                 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1214                break;
1215        case CHIP_PITCAIRN:
1216                radeon_program_register_sequence(rdev,
1217                                                 pitcairn_golden_registers,
1218                                                 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1219                radeon_program_register_sequence(rdev,
1220                                                 pitcairn_golden_rlc_registers,
1221                                                 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1222                radeon_program_register_sequence(rdev,
1223                                                 pitcairn_mgcg_cgcg_init,
1224                                                 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1225                break;
1226        case CHIP_VERDE:
1227                radeon_program_register_sequence(rdev,
1228                                                 verde_golden_registers,
1229                                                 (const u32)ARRAY_SIZE(verde_golden_registers));
1230                radeon_program_register_sequence(rdev,
1231                                                 verde_golden_rlc_registers,
1232                                                 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1233                radeon_program_register_sequence(rdev,
1234                                                 verde_mgcg_cgcg_init,
1235                                                 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1236                radeon_program_register_sequence(rdev,
1237                                                 verde_pg_init,
1238                                                 (const u32)ARRAY_SIZE(verde_pg_init));
1239                break;
1240        case CHIP_OLAND:
1241                radeon_program_register_sequence(rdev,
1242                                                 oland_golden_registers,
1243                                                 (const u32)ARRAY_SIZE(oland_golden_registers));
1244                radeon_program_register_sequence(rdev,
1245                                                 oland_golden_rlc_registers,
1246                                                 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1247                radeon_program_register_sequence(rdev,
1248                                                 oland_mgcg_cgcg_init,
1249                                                 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1250                break;
1251        case CHIP_HAINAN:
1252                radeon_program_register_sequence(rdev,
1253                                                 hainan_golden_registers,
1254                                                 (const u32)ARRAY_SIZE(hainan_golden_registers));
1255                radeon_program_register_sequence(rdev,
1256                                                 hainan_golden_registers2,
1257                                                 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1258                radeon_program_register_sequence(rdev,
1259                                                 hainan_mgcg_cgcg_init,
1260                                                 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1261                break;
1262        default:
1263                break;
1264        }
1265}
1266
1267/**
1268 * si_get_allowed_info_register - fetch the register for the info ioctl
1269 *
1270 * @rdev: radeon_device pointer
1271 * @reg: register offset in bytes
1272 * @val: register value
1273 *
1274 * Returns 0 for success or -EINVAL for an invalid register
1275 *
1276 */
1277int si_get_allowed_info_register(struct radeon_device *rdev,
1278                                 u32 reg, u32 *val)
1279{
1280        switch (reg) {
1281        case GRBM_STATUS:
1282        case GRBM_STATUS2:
1283        case GRBM_STATUS_SE0:
1284        case GRBM_STATUS_SE1:
1285        case SRBM_STATUS:
1286        case SRBM_STATUS2:
1287        case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1288        case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1289        case UVD_STATUS:
1290                *val = RREG32(reg);
1291                return 0;
1292        default:
1293                return -EINVAL;
1294        }
1295}
1296
1297#define PCIE_BUS_CLK                10000
1298#define TCLK                        (PCIE_BUS_CLK / 10)
1299
1300/**
1301 * si_get_xclk - get the xclk
1302 *
1303 * @rdev: radeon_device pointer
1304 *
1305 * Returns the reference clock used by the gfx engine
1306 * (SI).
1307 */
1308u32 si_get_xclk(struct radeon_device *rdev)
1309{
1310        u32 reference_clock = rdev->clock.spll.reference_freq;
1311        u32 tmp;
1312
1313        tmp = RREG32(CG_CLKPIN_CNTL_2);
1314        if (tmp & MUX_TCLK_TO_XCLK)
1315                return TCLK;
1316
1317        tmp = RREG32(CG_CLKPIN_CNTL);
1318        if (tmp & XTALIN_DIVIDE)
1319                return reference_clock / 4;
1320
1321        return reference_clock;
1322}
1323
1324/* get temperature in millidegrees */
1325int si_get_temp(struct radeon_device *rdev)
1326{
1327        u32 temp;
1328        int actual_temp = 0;
1329
1330        temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1331                CTF_TEMP_SHIFT;
1332
1333        if (temp & 0x200)
1334                actual_temp = 255;
1335        else
1336                actual_temp = temp & 0x1ff;
1337
1338        actual_temp = (actual_temp * 1000);
1339
1340        return actual_temp;
1341}
1342
1343#define TAHITI_IO_MC_REGS_SIZE 36
1344
1345static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1346        {0x0000006f, 0x03044000},
1347        {0x00000070, 0x0480c018},
1348        {0x00000071, 0x00000040},
1349        {0x00000072, 0x01000000},
1350        {0x00000074, 0x000000ff},
1351        {0x00000075, 0x00143400},
1352        {0x00000076, 0x08ec0800},
1353        {0x00000077, 0x040000cc},
1354        {0x00000079, 0x00000000},
1355        {0x0000007a, 0x21000409},
1356        {0x0000007c, 0x00000000},
1357        {0x0000007d, 0xe8000000},
1358        {0x0000007e, 0x044408a8},
1359        {0x0000007f, 0x00000003},
1360        {0x00000080, 0x00000000},
1361        {0x00000081, 0x01000000},
1362        {0x00000082, 0x02000000},
1363        {0x00000083, 0x00000000},
1364        {0x00000084, 0xe3f3e4f4},
1365        {0x00000085, 0x00052024},
1366        {0x00000087, 0x00000000},
1367        {0x00000088, 0x66036603},
1368        {0x00000089, 0x01000000},
1369        {0x0000008b, 0x1c0a0000},
1370        {0x0000008c, 0xff010000},
1371        {0x0000008e, 0xffffefff},
1372        {0x0000008f, 0xfff3efff},
1373        {0x00000090, 0xfff3efbf},
1374        {0x00000094, 0x00101101},
1375        {0x00000095, 0x00000fff},
1376        {0x00000096, 0x00116fff},
1377        {0x00000097, 0x60010000},
1378        {0x00000098, 0x10010000},
1379        {0x00000099, 0x00006000},
1380        {0x0000009a, 0x00001000},
1381        {0x0000009f, 0x00a77400}
1382};
1383
1384static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1385        {0x0000006f, 0x03044000},
1386        {0x00000070, 0x0480c018},
1387        {0x00000071, 0x00000040},
1388        {0x00000072, 0x01000000},
1389        {0x00000074, 0x000000ff},
1390        {0x00000075, 0x00143400},
1391        {0x00000076, 0x08ec0800},
1392        {0x00000077, 0x040000cc},
1393        {0x00000079, 0x00000000},
1394        {0x0000007a, 0x21000409},
1395        {0x0000007c, 0x00000000},
1396        {0x0000007d, 0xe8000000},
1397        {0x0000007e, 0x044408a8},
1398        {0x0000007f, 0x00000003},
1399        {0x00000080, 0x00000000},
1400        {0x00000081, 0x01000000},
1401        {0x00000082, 0x02000000},
1402        {0x00000083, 0x00000000},
1403        {0x00000084, 0xe3f3e4f4},
1404        {0x00000085, 0x00052024},
1405        {0x00000087, 0x00000000},
1406        {0x00000088, 0x66036603},
1407        {0x00000089, 0x01000000},
1408        {0x0000008b, 0x1c0a0000},
1409        {0x0000008c, 0xff010000},
1410        {0x0000008e, 0xffffefff},
1411        {0x0000008f, 0xfff3efff},
1412        {0x00000090, 0xfff3efbf},
1413        {0x00000094, 0x00101101},
1414        {0x00000095, 0x00000fff},
1415        {0x00000096, 0x00116fff},
1416        {0x00000097, 0x60010000},
1417        {0x00000098, 0x10010000},
1418        {0x00000099, 0x00006000},
1419        {0x0000009a, 0x00001000},
1420        {0x0000009f, 0x00a47400}
1421};
1422
1423static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1424        {0x0000006f, 0x03044000},
1425        {0x00000070, 0x0480c018},
1426        {0x00000071, 0x00000040},
1427        {0x00000072, 0x01000000},
1428        {0x00000074, 0x000000ff},
1429        {0x00000075, 0x00143400},
1430        {0x00000076, 0x08ec0800},
1431        {0x00000077, 0x040000cc},
1432        {0x00000079, 0x00000000},
1433        {0x0000007a, 0x21000409},
1434        {0x0000007c, 0x00000000},
1435        {0x0000007d, 0xe8000000},
1436        {0x0000007e, 0x044408a8},
1437        {0x0000007f, 0x00000003},
1438        {0x00000080, 0x00000000},
1439        {0x00000081, 0x01000000},
1440        {0x00000082, 0x02000000},
1441        {0x00000083, 0x00000000},
1442        {0x00000084, 0xe3f3e4f4},
1443        {0x00000085, 0x00052024},
1444        {0x00000087, 0x00000000},
1445        {0x00000088, 0x66036603},
1446        {0x00000089, 0x01000000},
1447        {0x0000008b, 0x1c0a0000},
1448        {0x0000008c, 0xff010000},
1449        {0x0000008e, 0xffffefff},
1450        {0x0000008f, 0xfff3efff},
1451        {0x00000090, 0xfff3efbf},
1452        {0x00000094, 0x00101101},
1453        {0x00000095, 0x00000fff},
1454        {0x00000096, 0x00116fff},
1455        {0x00000097, 0x60010000},
1456        {0x00000098, 0x10010000},
1457        {0x00000099, 0x00006000},
1458        {0x0000009a, 0x00001000},
1459        {0x0000009f, 0x00a37400}
1460};
1461
1462static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1463        {0x0000006f, 0x03044000},
1464        {0x00000070, 0x0480c018},
1465        {0x00000071, 0x00000040},
1466        {0x00000072, 0x01000000},
1467        {0x00000074, 0x000000ff},
1468        {0x00000075, 0x00143400},
1469        {0x00000076, 0x08ec0800},
1470        {0x00000077, 0x040000cc},
1471        {0x00000079, 0x00000000},
1472        {0x0000007a, 0x21000409},
1473        {0x0000007c, 0x00000000},
1474        {0x0000007d, 0xe8000000},
1475        {0x0000007e, 0x044408a8},
1476        {0x0000007f, 0x00000003},
1477        {0x00000080, 0x00000000},
1478        {0x00000081, 0x01000000},
1479        {0x00000082, 0x02000000},
1480        {0x00000083, 0x00000000},
1481        {0x00000084, 0xe3f3e4f4},
1482        {0x00000085, 0x00052024},
1483        {0x00000087, 0x00000000},
1484        {0x00000088, 0x66036603},
1485        {0x00000089, 0x01000000},
1486        {0x0000008b, 0x1c0a0000},
1487        {0x0000008c, 0xff010000},
1488        {0x0000008e, 0xffffefff},
1489        {0x0000008f, 0xfff3efff},
1490        {0x00000090, 0xfff3efbf},
1491        {0x00000094, 0x00101101},
1492        {0x00000095, 0x00000fff},
1493        {0x00000096, 0x00116fff},
1494        {0x00000097, 0x60010000},
1495        {0x00000098, 0x10010000},
1496        {0x00000099, 0x00006000},
1497        {0x0000009a, 0x00001000},
1498        {0x0000009f, 0x00a17730}
1499};
1500
1501static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1502        {0x0000006f, 0x03044000},
1503        {0x00000070, 0x0480c018},
1504        {0x00000071, 0x00000040},
1505        {0x00000072, 0x01000000},
1506        {0x00000074, 0x000000ff},
1507        {0x00000075, 0x00143400},
1508        {0x00000076, 0x08ec0800},
1509        {0x00000077, 0x040000cc},
1510        {0x00000079, 0x00000000},
1511        {0x0000007a, 0x21000409},
1512        {0x0000007c, 0x00000000},
1513        {0x0000007d, 0xe8000000},
1514        {0x0000007e, 0x044408a8},
1515        {0x0000007f, 0x00000003},
1516        {0x00000080, 0x00000000},
1517        {0x00000081, 0x01000000},
1518        {0x00000082, 0x02000000},
1519        {0x00000083, 0x00000000},
1520        {0x00000084, 0xe3f3e4f4},
1521        {0x00000085, 0x00052024},
1522        {0x00000087, 0x00000000},
1523        {0x00000088, 0x66036603},
1524        {0x00000089, 0x01000000},
1525        {0x0000008b, 0x1c0a0000},
1526        {0x0000008c, 0xff010000},
1527        {0x0000008e, 0xffffefff},
1528        {0x0000008f, 0xfff3efff},
1529        {0x00000090, 0xfff3efbf},
1530        {0x00000094, 0x00101101},
1531        {0x00000095, 0x00000fff},
1532        {0x00000096, 0x00116fff},
1533        {0x00000097, 0x60010000},
1534        {0x00000098, 0x10010000},
1535        {0x00000099, 0x00006000},
1536        {0x0000009a, 0x00001000},
1537        {0x0000009f, 0x00a07730}
1538};
1539
1540/* ucode loading */
1541int si_mc_load_microcode(struct radeon_device *rdev)
1542{
1543        const __be32 *fw_data = NULL;
1544        const __le32 *new_fw_data = NULL;
1545        u32 running, blackout = 0;
1546        u32 *io_mc_regs = NULL;
1547        const __le32 *new_io_mc_regs = NULL;
1548        int i, regs_size, ucode_size;
1549
1550        if (!rdev->mc_fw)
1551                return -EINVAL;
1552
1553        if (rdev->new_fw) {
1554                const struct mc_firmware_header_v1_0 *hdr =
1555                        (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1556
1557                radeon_ucode_print_mc_hdr(&hdr->header);
1558                regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1559                new_io_mc_regs = (const __le32 *)
1560                        (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1561                ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1562                new_fw_data = (const __le32 *)
1563                        (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1564        } else {
1565                ucode_size = rdev->mc_fw->size / 4;
1566
1567                switch (rdev->family) {
1568                case CHIP_TAHITI:
1569                        io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1570                        regs_size = TAHITI_IO_MC_REGS_SIZE;
1571                        break;
1572                case CHIP_PITCAIRN:
1573                        io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1574                        regs_size = TAHITI_IO_MC_REGS_SIZE;
1575                        break;
1576                case CHIP_VERDE:
1577                default:
1578                        io_mc_regs = (u32 *)&verde_io_mc_regs;
1579                        regs_size = TAHITI_IO_MC_REGS_SIZE;
1580                        break;
1581                case CHIP_OLAND:
1582                        io_mc_regs = (u32 *)&oland_io_mc_regs;
1583                        regs_size = TAHITI_IO_MC_REGS_SIZE;
1584                        break;
1585                case CHIP_HAINAN:
1586                        io_mc_regs = (u32 *)&hainan_io_mc_regs;
1587                        regs_size = TAHITI_IO_MC_REGS_SIZE;
1588                        break;
1589                }
1590                fw_data = (const __be32 *)rdev->mc_fw->data;
1591        }
1592
1593        running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1594
1595        if (running == 0) {
1596                if (running) {
1597                        blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1598                        WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1599                }
1600
1601                /* reset the engine and set to writable */
1602                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1603                WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1604
1605                /* load mc io regs */
1606                for (i = 0; i < regs_size; i++) {
1607                        if (rdev->new_fw) {
1608                                WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1609                                WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1610                        } else {
1611                                WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1612                                WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1613                        }
1614                }
1615                /* load the MC ucode */
1616                for (i = 0; i < ucode_size; i++) {
1617                        if (rdev->new_fw)
1618                                WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1619                        else
1620                                WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1621                }
1622
1623                /* put the engine back into the active state */
1624                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1625                WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1626                WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1627
1628                /* wait for training to complete */
1629                for (i = 0; i < rdev->usec_timeout; i++) {
1630                        if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1631                                break;
1632                        udelay(1);
1633                }
1634                for (i = 0; i < rdev->usec_timeout; i++) {
1635                        if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1636                                break;
1637                        udelay(1);
1638                }
1639
1640                if (running)
1641                        WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1642        }
1643
1644        return 0;
1645}
1646
1647static int si_init_microcode(struct radeon_device *rdev)
1648{
1649        const char *chip_name;
1650        const char *new_chip_name;
1651        size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1652        size_t smc_req_size, mc2_req_size;
1653        char fw_name[30];
1654        int err;
1655        int new_fw = 0;
1656
1657        DRM_DEBUG("\n");
1658
1659        switch (rdev->family) {
1660        case CHIP_TAHITI:
1661                chip_name = "TAHITI";
1662                new_chip_name = "tahiti";
1663                pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1664                me_req_size = SI_PM4_UCODE_SIZE * 4;
1665                ce_req_size = SI_CE_UCODE_SIZE * 4;
1666                rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1667                mc_req_size = SI_MC_UCODE_SIZE * 4;
1668                mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1669                smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1670                break;
1671        case CHIP_PITCAIRN:
1672                chip_name = "PITCAIRN";
1673                new_chip_name = "pitcairn";
1674                pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1675                me_req_size = SI_PM4_UCODE_SIZE * 4;
1676                ce_req_size = SI_CE_UCODE_SIZE * 4;
1677                rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1678                mc_req_size = SI_MC_UCODE_SIZE * 4;
1679                mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1680                smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1681                break;
1682        case CHIP_VERDE:
1683                chip_name = "VERDE";
1684                new_chip_name = "verde";
1685                pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1686                me_req_size = SI_PM4_UCODE_SIZE * 4;
1687                ce_req_size = SI_CE_UCODE_SIZE * 4;
1688                rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1689                mc_req_size = SI_MC_UCODE_SIZE * 4;
1690                mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1691                smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1692                break;
1693        case CHIP_OLAND:
1694                chip_name = "OLAND";
1695                new_chip_name = "oland";
1696                pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1697                me_req_size = SI_PM4_UCODE_SIZE * 4;
1698                ce_req_size = SI_CE_UCODE_SIZE * 4;
1699                rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1700                mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1701                smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1702                break;
1703        case CHIP_HAINAN:
1704                chip_name = "HAINAN";
1705                new_chip_name = "hainan";
1706                pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1707                me_req_size = SI_PM4_UCODE_SIZE * 4;
1708                ce_req_size = SI_CE_UCODE_SIZE * 4;
1709                rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1710                mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1711                smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1712                break;
1713        default: BUG();
1714        }
1715
1716        DRM_INFO("Loading %s Microcode\n", new_chip_name);
1717
1718        snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1719        err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1720        if (err) {
1721                snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1722                err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1723                if (err)
1724                        goto out;
1725                if (rdev->pfp_fw->size != pfp_req_size) {
1726                        printk(KERN_ERR
1727                               "si_cp: Bogus length %zu in firmware \"%s\"\n",
1728                               rdev->pfp_fw->size, fw_name);
1729                        err = -EINVAL;
1730                        goto out;
1731                }
1732        } else {
1733                err = radeon_ucode_validate(rdev->pfp_fw);
1734                if (err) {
1735                        printk(KERN_ERR
1736                               "si_cp: validation failed for firmware \"%s\"\n",
1737                               fw_name);
1738                        goto out;
1739                } else {
1740                        new_fw++;
1741                }
1742        }
1743
1744        snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
1745        err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1746        if (err) {
1747                snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1748                err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1749                if (err)
1750                        goto out;
1751                if (rdev->me_fw->size != me_req_size) {
1752                        printk(KERN_ERR
1753                               "si_cp: Bogus length %zu in firmware \"%s\"\n",
1754                               rdev->me_fw->size, fw_name);
1755                        err = -EINVAL;
1756                }
1757        } else {
1758                err = radeon_ucode_validate(rdev->me_fw);
1759                if (err) {
1760                        printk(KERN_ERR
1761                               "si_cp: validation failed for firmware \"%s\"\n",
1762                               fw_name);
1763                        goto out;
1764                } else {
1765                        new_fw++;
1766                }
1767        }
1768
1769        snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
1770        err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1771        if (err) {
1772                snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1773                err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1774                if (err)
1775                        goto out;
1776                if (rdev->ce_fw->size != ce_req_size) {
1777                        printk(KERN_ERR
1778                               "si_cp: Bogus length %zu in firmware \"%s\"\n",
1779                               rdev->ce_fw->size, fw_name);
1780                        err = -EINVAL;
1781                }
1782        } else {
1783                err = radeon_ucode_validate(rdev->ce_fw);
1784                if (err) {
1785                        printk(KERN_ERR
1786                               "si_cp: validation failed for firmware \"%s\"\n",
1787                               fw_name);
1788                        goto out;
1789                } else {
1790                        new_fw++;
1791                }
1792        }
1793
1794        snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
1795        err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1796        if (err) {
1797                snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1798                err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1799                if (err)
1800                        goto out;
1801                if (rdev->rlc_fw->size != rlc_req_size) {
1802                        printk(KERN_ERR
1803                               "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1804                               rdev->rlc_fw->size, fw_name);
1805                        err = -EINVAL;
1806                }
1807        } else {
1808                err = radeon_ucode_validate(rdev->rlc_fw);
1809                if (err) {
1810                        printk(KERN_ERR
1811                               "si_cp: validation failed for firmware \"%s\"\n",
1812                               fw_name);
1813                        goto out;
1814                } else {
1815                        new_fw++;
1816                }
1817        }
1818
1819        snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
1820        err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1821        if (err) {
1822                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1823                err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1824                if (err) {
1825                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1826                        err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1827                        if (err)
1828                                goto out;
1829                }
1830                if ((rdev->mc_fw->size != mc_req_size) &&
1831                    (rdev->mc_fw->size != mc2_req_size)) {
1832                        printk(KERN_ERR
1833                               "si_mc: Bogus length %zu in firmware \"%s\"\n",
1834                               rdev->mc_fw->size, fw_name);
1835                        err = -EINVAL;
1836                }
1837                DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1838        } else {
1839                err = radeon_ucode_validate(rdev->mc_fw);
1840                if (err) {
1841                        printk(KERN_ERR
1842                               "si_cp: validation failed for firmware \"%s\"\n",
1843                               fw_name);
1844                        goto out;
1845                } else {
1846                        new_fw++;
1847                }
1848        }
1849
1850        snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
1851        err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1852        if (err) {
1853                snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1854                err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1855                if (err) {
1856                        printk(KERN_ERR
1857                               "smc: error loading firmware \"%s\"\n",
1858                               fw_name);
1859                        release_firmware(rdev->smc_fw);
1860                        rdev->smc_fw = NULL;
1861                        err = 0;
1862                } else if (rdev->smc_fw->size != smc_req_size) {
1863                        printk(KERN_ERR
1864                               "si_smc: Bogus length %zu in firmware \"%s\"\n",
1865                               rdev->smc_fw->size, fw_name);
1866                        err = -EINVAL;
1867                }
1868        } else {
1869                err = radeon_ucode_validate(rdev->smc_fw);
1870                if (err) {
1871                        printk(KERN_ERR
1872                               "si_cp: validation failed for firmware \"%s\"\n",
1873                               fw_name);
1874                        goto out;
1875                } else {
1876                        new_fw++;
1877                }
1878        }
1879
1880        if (new_fw == 0) {
1881                rdev->new_fw = false;
1882        } else if (new_fw < 6) {
1883                printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
1884                err = -EINVAL;
1885        } else {
1886                rdev->new_fw = true;
1887        }
1888out:
1889        if (err) {
1890                if (err != -EINVAL)
1891                        printk(KERN_ERR
1892                               "si_cp: Failed to load firmware \"%s\"\n",
1893                               fw_name);
1894                release_firmware(rdev->pfp_fw);
1895                rdev->pfp_fw = NULL;
1896                release_firmware(rdev->me_fw);
1897                rdev->me_fw = NULL;
1898                release_firmware(rdev->ce_fw);
1899                rdev->ce_fw = NULL;
1900                release_firmware(rdev->rlc_fw);
1901                rdev->rlc_fw = NULL;
1902                release_firmware(rdev->mc_fw);
1903                rdev->mc_fw = NULL;
1904                release_firmware(rdev->smc_fw);
1905                rdev->smc_fw = NULL;
1906        }
1907        return err;
1908}
1909
1910/* watermark setup */
1911static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1912                                   struct radeon_crtc *radeon_crtc,
1913                                   struct drm_display_mode *mode,
1914                                   struct drm_display_mode *other_mode)
1915{
1916        u32 tmp, buffer_alloc, i;
1917        u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1918        /*
1919         * Line Buffer Setup
1920         * There are 3 line buffers, each one shared by 2 display controllers.
1921         * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1922         * the display controllers.  The paritioning is done via one of four
1923         * preset allocations specified in bits 21:20:
1924         *  0 - half lb
1925         *  2 - whole lb, other crtc must be disabled
1926         */
1927        /* this can get tricky if we have two large displays on a paired group
1928         * of crtcs.  Ideally for multiple large displays we'd assign them to
1929         * non-linked crtcs for maximum line buffer allocation.
1930         */
1931        if (radeon_crtc->base.enabled && mode) {
1932                if (other_mode) {
1933                        tmp = 0; /* 1/2 */
1934                        buffer_alloc = 1;
1935                } else {
1936                        tmp = 2; /* whole */
1937                        buffer_alloc = 2;
1938                }
1939        } else {
1940                tmp = 0;
1941                buffer_alloc = 0;
1942        }
1943
1944        WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1945               DC_LB_MEMORY_CONFIG(tmp));
1946
1947        WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1948               DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1949        for (i = 0; i < rdev->usec_timeout; i++) {
1950                if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1951                    DMIF_BUFFERS_ALLOCATED_COMPLETED)
1952                        break;
1953                udelay(1);
1954        }
1955
1956        if (radeon_crtc->base.enabled && mode) {
1957                switch (tmp) {
1958                case 0:
1959                default:
1960                        return 4096 * 2;
1961                case 2:
1962                        return 8192 * 2;
1963                }
1964        }
1965
1966        /* controller not enabled, so no lb used */
1967        return 0;
1968}
1969
1970static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1971{
1972        u32 tmp = RREG32(MC_SHARED_CHMAP);
1973
1974        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1975        case 0:
1976        default:
1977                return 1;
1978        case 1:
1979                return 2;
1980        case 2:
1981                return 4;
1982        case 3:
1983                return 8;
1984        case 4:
1985                return 3;
1986        case 5:
1987                return 6;
1988        case 6:
1989                return 10;
1990        case 7:
1991                return 12;
1992        case 8:
1993                return 16;
1994        }
1995}
1996
1997struct dce6_wm_params {
1998        u32 dram_channels; /* number of dram channels */
1999        u32 yclk;          /* bandwidth per dram data pin in kHz */
2000        u32 sclk;          /* engine clock in kHz */

2001        u32 disp_clk;      /* display clock in kHz */
2002        u32 src_width;     /* viewport width */
2003        u32 active_time;   /* active display time in ns */
2004        u32 blank_time;    /* blank time in ns */
2005        bool interlaced;    /* mode is interlaced */
2006        fixed20_12 vsc;    /* vertical scale ratio */
2007        u32 num_heads;     /* number of active crtcs */
2008        u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2009        u32 lb_size;       /* line buffer allocated to pipe */
2010        u32 vtaps;         /* vertical scaler taps */
2011};
2012
2013static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
2014{
2015        /* Calculate raw DRAM Bandwidth */
2016        fixed20_12 dram_efficiency; /* 0.7 */
2017        fixed20_12 yclk, dram_channels, bandwidth;
2018        fixed20_12 a;
2019
2020        a.full = dfixed_const(1000);
2021        yclk.full = dfixed_const(wm->yclk);
2022        yclk.full = dfixed_div(yclk, a);
2023        dram_channels.full = dfixed_const(wm->dram_channels * 4);
2024        a.full = dfixed_const(10);
2025        dram_efficiency.full = dfixed_const(7);
2026        dram_efficiency.full = dfixed_div(dram_efficiency, a);
2027        bandwidth.full = dfixed_mul(dram_channels, yclk);
2028        bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2029
2030        return dfixed_trunc(bandwidth);
2031}
2032
2033static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2034{
2035        /* Calculate DRAM Bandwidth and the part allocated to display. */
2036        fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2037        fixed20_12 yclk, dram_channels, bandwidth;
2038        fixed20_12 a;
2039
2040        a.full = dfixed_const(1000);
2041        yclk.full = dfixed_const(wm->yclk);
2042        yclk.full = dfixed_div(yclk, a);
2043        dram_channels.full = dfixed_const(wm->dram_channels * 4);
2044        a.full = dfixed_const(10);
2045        disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2046        disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2047        bandwidth.full = dfixed_mul(dram_channels, yclk);
2048        bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2049
2050        return dfixed_trunc(bandwidth);
2051}
2052
2053static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2054{
2055        /* Calculate the display Data return Bandwidth */
2056        fixed20_12 return_efficiency; /* 0.8 */
2057        fixed20_12 sclk, bandwidth;
2058        fixed20_12 a;
2059
2060        a.full = dfixed_const(1000);
2061        sclk.full = dfixed_const(wm->sclk);
2062        sclk.full = dfixed_div(sclk, a);
2063        a.full = dfixed_const(10);
2064        return_efficiency.full = dfixed_const(8);
2065        return_efficiency.full = dfixed_div(return_efficiency, a);
2066        a.full = dfixed_const(32);
2067        bandwidth.full = dfixed_mul(a, sclk);
2068        bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2069
2070        return dfixed_trunc(bandwidth);
2071}
2072
2073static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2074{
2075        return 32;
2076}
2077
2078static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2079{
2080        /* Calculate the DMIF Request Bandwidth */
2081        fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2082        fixed20_12 disp_clk, sclk, bandwidth;
2083        fixed20_12 a, b1, b2;
2084        u32 min_bandwidth;
2085
2086        a.full = dfixed_const(1000);
2087        disp_clk.full = dfixed_const(wm->disp_clk);
2088        disp_clk.full = dfixed_div(disp_clk, a);
2089        a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2090        b1.full = dfixed_mul(a, disp_clk);
2091
2092        a.full = dfixed_const(1000);
2093        sclk.full = dfixed_const(wm->sclk);
2094        sclk.full = dfixed_div(sclk, a);
2095        a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2096        b2.full = dfixed_mul(a, sclk);
2097
2098        a.full = dfixed_const(10);
2099        disp_clk_request_efficiency.full = dfixed_const(8);
2100        disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2101
2102        min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2103
2104        a.full = dfixed_const(min_bandwidth);
2105        bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2106
2107        return dfixed_trunc(bandwidth);
2108}
2109
2110static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2111{
2112        /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2113        u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2114        u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2115        u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2116
2117        return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2118}
2119
2120static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2121{
2122        /* Calculate the display mode Average Bandwidth
2123         * DisplayMode should contain the source and destination dimensions,
2124         * timing, etc.
2125         */
2126        fixed20_12 bpp;
2127        fixed20_12 line_time;
2128        fixed20_12 src_width;
2129        fixed20_12 bandwidth;
2130        fixed20_12 a;
2131
2132        a.full = dfixed_const(1000);
2133        line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2134        line_time.full = dfixed_div(line_time, a);
2135        bpp.full = dfixed_const(wm->bytes_per_pixel);
2136        src_width.full = dfixed_const(wm->src_width);
2137        bandwidth.full = dfixed_mul(src_width, bpp);
2138        bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2139        bandwidth.full = dfixed_div(bandwidth, line_time);
2140
2141        return dfixed_trunc(bandwidth);
2142}
2143
2144static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2145{
2146        /* First calcualte the latency in ns */
2147        u32 mc_latency = 2000; /* 2000 ns. */
2148        u32 available_bandwidth = dce6_available_bandwidth(wm);
2149        u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2150        u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2151        u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2152        u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2153                (wm->num_heads * cursor_line_pair_return_time);
2154        u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2155        u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2156        u32 tmp, dmif_size = 12288;
2157        fixed20_12 a, b, c;
2158
2159        if (wm->num_heads == 0)
2160                return 0;
2161
2162        a.full = dfixed_const(2);
2163        b.full = dfixed_const(1);
2164        if ((wm->vsc.full > a.full) ||
2165            ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2166            (wm->vtaps >= 5) ||
2167            ((wm->vsc.full >= a.full) && wm->interlaced))
2168                max_src_lines_per_dst_line = 4;
2169        else
2170                max_src_lines_per_dst_line = 2;
2171
2172        a.full = dfixed_const(available_bandwidth);
2173        b.full = dfixed_const(wm->num_heads);
2174        a.full = dfixed_div(a, b);
2175
2176        b.full = dfixed_const(mc_latency + 512);
2177        c.full = dfixed_const(wm->disp_clk);
2178        b.full = dfixed_div(b, c);
2179
2180        c.full = dfixed_const(dmif_size);
2181        b.full = dfixed_div(c, b);
2182
2183        tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2184
2185        b.full = dfixed_const(1000);
2186        c.full = dfixed_const(wm->disp_clk);
2187        b.full = dfixed_div(c, b);
2188        c.full = dfixed_const(wm->bytes_per_pixel);
2189        b.full = dfixed_mul(b, c);
2190
2191        lb_fill_bw = min(tmp, dfixed_trunc(b));
2192
2193        a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2194        b.full = dfixed_const(1000);
2195        c.full = dfixed_const(lb_fill_bw);
2196        b.full = dfixed_div(c, b);
2197        a.full = dfixed_div(a, b);
2198        line_fill_time = dfixed_trunc(a);
2199
2200        if (line_fill_time < wm->active_time)
2201                return latency;
2202        else
2203                return latency + (line_fill_time - wm->active_time);
2204
2205}
2206
2207static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2208{
2209        if (dce6_average_bandwidth(wm) <=
2210            (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2211                return true;
2212        else
2213                return false;
2214};
2215
2216static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2217{
2218        if (dce6_average_bandwidth(wm) <=
2219            (dce6_available_bandwidth(wm) / wm->num_heads))
2220                return true;
2221        else
2222                return false;
2223};
2224
2225static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2226{
2227        u32 lb_partitions = wm->lb_size / wm->src_width;
2228        u32 line_time = wm->active_time + wm->blank_time;
2229        u32 latency_tolerant_lines;
2230        u32 latency_hiding;
2231        fixed20_12 a;
2232
2233        a.full = dfixed_const(1);
2234        if (wm->vsc.full > a.full)
2235                latency_tolerant_lines = 1;
2236        else {
2237                if (lb_partitions <= (wm->vtaps + 1))
2238                        latency_tolerant_lines = 1;
2239                else
2240                        latency_tolerant_lines = 2;
2241        }
2242
2243        latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2244
2245        if (dce6_latency_watermark(wm) <= latency_hiding)
2246                return true;
2247        else
2248                return false;
2249}
2250
2251static void dce6_program_watermarks(struct radeon_device *rdev,
2252                                         struct radeon_crtc *radeon_crtc,
2253                                         u32 lb_size, u32 num_heads)
2254{
2255        struct drm_display_mode *mode = &radeon_crtc->base.mode;
2256        struct dce6_wm_params wm_low, wm_high;
2257        u32 dram_channels;
2258        u32 pixel_period;
2259        u32 line_time = 0;
2260        u32 latency_watermark_a = 0, latency_watermark_b = 0;
2261        u32 priority_a_mark = 0, priority_b_mark = 0;
2262        u32 priority_a_cnt = PRIORITY_OFF;
2263        u32 priority_b_cnt = PRIORITY_OFF;
2264        u32 tmp, arb_control3;
2265        fixed20_12 a, b, c;
2266
2267        if (radeon_crtc->base.enabled && num_heads && mode) {
2268                pixel_period = 1000000 / (u32)mode->clock;
2269                line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2270                priority_a_cnt = 0;
2271                priority_b_cnt = 0;
2272
2273                if (rdev->family == CHIP_ARUBA)
2274                        dram_channels = evergreen_get_number_of_dram_channels(rdev);
2275                else
2276                        dram_channels = si_get_number_of_dram_channels(rdev);
2277
2278                /* watermark for high clocks */
2279                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2280                        wm_high.yclk =
2281                                radeon_dpm_get_mclk(rdev, false) * 10;
2282                        wm_high.sclk =
2283                                radeon_dpm_get_sclk(rdev, false) * 10;
2284                } else {
2285                        wm_high.yclk = rdev->pm.current_mclk * 10;
2286                        wm_high.sclk = rdev->pm.current_sclk * 10;
2287                }
2288
2289                wm_high.disp_clk = mode->clock;
2290                wm_high.src_width = mode->crtc_hdisplay;
2291                wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2292                wm_high.blank_time = line_time - wm_high.active_time;
2293                wm_high.interlaced = false;
2294                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2295                        wm_high.interlaced = true;
2296                wm_high.vsc = radeon_crtc->vsc;
2297                wm_high.vtaps = 1;
2298                if (radeon_crtc->rmx_type != RMX_OFF)
2299                        wm_high.vtaps = 2;
2300                wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2301                wm_high.lb_size = lb_size;
2302                wm_high.dram_channels = dram_channels;
2303                wm_high.num_heads = num_heads;
2304
2305                /* watermark for low clocks */
2306                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2307                        wm_low.yclk =
2308                                radeon_dpm_get_mclk(rdev, true) * 10;
2309                        wm_low.sclk =
2310                                radeon_dpm_get_sclk(rdev, true) * 10;
2311                } else {
2312                        wm_low.yclk = rdev->pm.current_mclk * 10;
2313                        wm_low.sclk = rdev->pm.current_sclk * 10;
2314                }
2315
2316                wm_low.disp_clk = mode->clock;
2317                wm_low.src_width = mode->crtc_hdisplay;
2318                wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2319                wm_low.blank_time = line_time - wm_low.active_time;
2320                wm_low.interlaced = false;
2321                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2322                        wm_low.interlaced = true;
2323                wm_low.vsc = radeon_crtc->vsc;
2324                wm_low.vtaps = 1;
2325                if (radeon_crtc->rmx_type != RMX_OFF)
2326                        wm_low.vtaps = 2;
2327                wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2328                wm_low.lb_size = lb_size;
2329                wm_low.dram_channels = dram_channels;
2330                wm_low.num_heads = num_heads;
2331
2332                /* set for high clocks */
2333                latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2334                /* set for low clocks */
2335                latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2336
2337                /* possibly force display priority to high */
2338                /* should really do this at mode validation time... */
2339                if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2340                    !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2341                    !dce6_check_latency_hiding(&wm_high) ||
2342                    (rdev->disp_priority == 2)) {
2343                        DRM_DEBUG_KMS("force priority to high\n");
2344                        priority_a_cnt |= PRIORITY_ALWAYS_ON;
2345                        priority_b_cnt |= PRIORITY_ALWAYS_ON;
2346                }
2347                if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2348                    !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2349                    !dce6_check_latency_hiding(&wm_low) ||
2350                    (rdev->disp_priority == 2)) {
2351                        DRM_DEBUG_KMS("force priority to high\n");
2352                        priority_a_cnt |= PRIORITY_ALWAYS_ON;
2353                        priority_b_cnt |= PRIORITY_ALWAYS_ON;
2354                }
2355
2356                a.full = dfixed_const(1000);
2357                b.full = dfixed_const(mode->clock);
2358                b.full = dfixed_div(b, a);
2359                c.full = dfixed_const(latency_watermark_a);
2360                c.full = dfixed_mul(c, b);
2361                c.full = dfixed_mul(c, radeon_crtc->hsc);
2362                c.full = dfixed_div(c, a);
2363                a.full = dfixed_const(16);
2364                c.full = dfixed_div(c, a);
2365                priority_a_mark = dfixed_trunc(c);
2366                priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2367
2368                a.full = dfixed_const(1000);
2369                b.full = dfixed_const(mode->clock);
2370                b.full = dfixed_div(b, a);
2371                c.full = dfixed_const(latency_watermark_b);
2372                c.full = dfixed_mul(c, b);
2373                c.full = dfixed_mul(c, radeon_crtc->hsc);
2374                c.full = dfixed_div(c, a);
2375                a.full = dfixed_const(16);
2376                c.full = dfixed_div(c, a);
2377                priority_b_mark = dfixed_trunc(c);
2378                priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2379
2380                /* Save number of lines the linebuffer leads before the scanout */
2381                radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2382        }
2383
2384        /* select wm A */
2385        arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2386        tmp = arb_control3;
2387        tmp &= ~LATENCY_WATERMARK_MASK(3);
2388        tmp |= LATENCY_WATERMARK_MASK(1);
2389        WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2390        WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2391               (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2392                LATENCY_HIGH_WATERMARK(line_time)));
2393        /* select wm B */
2394        tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2395        tmp &= ~LATENCY_WATERMARK_MASK(3);
2396        tmp |= LATENCY_WATERMARK_MASK(2);
2397        WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2398        WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2399               (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2400                LATENCY_HIGH_WATERMARK(line_time)));
2401        /* restore original selection */
2402        WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2403
2404        /* write the priority marks */
2405        WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2406        WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2407
2408        /* save values for DPM */
2409        radeon_crtc->line_time = line_time;
2410        radeon_crtc->wm_high = latency_watermark_a;
2411        radeon_crtc->wm_low = latency_watermark_b;
2412}
2413
2414void dce6_bandwidth_update(struct radeon_device *rdev)
2415{
2416        struct drm_display_mode *mode0 = NULL;
2417        struct drm_display_mode *mode1 = NULL;
2418        u32 num_heads = 0, lb_size;
2419        int i;
2420
2421        if (!rdev->mode_info.mode_config_initialized)
2422                return;
2423
2424        radeon_update_display_priority(rdev);
2425
2426        for (i = 0; i < rdev->num_crtc; i++) {
2427                if (rdev->mode_info.crtcs[i]->base.enabled)
2428                        num_heads++;
2429        }
2430        for (i = 0; i < rdev->num_crtc; i += 2) {
2431                mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2432                mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2433                lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2434                dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2435                lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2436                dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2437        }
2438}
2439
2440/*
2441 * Core functions
2442 */
2443static void si_tiling_mode_table_init(struct radeon_device *rdev)
2444{
2445        u32 *tile = rdev->config.si.tile_mode_array;
2446        const u32 num_tile_mode_states =
2447                        ARRAY_SIZE(rdev->config.si.tile_mode_array);
2448        u32 reg_offset, split_equal_to_row_size;
2449
2450        switch (rdev->config.si.mem_row_size_in_kb) {
2451        case 1:
2452                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2453                break;
2454        case 2:
2455        default:
2456                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2457                break;
2458        case 4:
2459                split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2460                break;
2461        }
2462
2463        for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2464                tile[reg_offset] = 0;
2465
2466        switch(rdev->family) {
2467        case CHIP_TAHITI:
2468        case CHIP_PITCAIRN:
2469                /* non-AA compressed depth or any compressed stencil */
2470                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2471                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2472                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2473                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2474                           NUM_BANKS(ADDR_SURF_16_BANK) |
2475                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2477                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2478                /* 2xAA/4xAA compressed depth only */
2479                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2480                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2481                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2482                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2483                           NUM_BANKS(ADDR_SURF_16_BANK) |
2484                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2486                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2487                /* 8xAA compressed depth only */
2488                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2489                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2490                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2491                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2492                           NUM_BANKS(ADDR_SURF_16_BANK) |
2493                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2494                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2495                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2496                /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2497                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2498                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2499                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2500                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2501                           NUM_BANKS(ADDR_SURF_16_BANK) |
2502                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2503                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2504                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2505                /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2506                tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2507                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2508                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2509                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2510                           NUM_BANKS(ADDR_SURF_16_BANK) |
2511                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2512                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2513                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2514                /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2515                tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2516                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2517                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2518                           TILE_SPLIT(split_equal_to_row_size) |
2519                           NUM_BANKS(ADDR_SURF_16_BANK) |
2520                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2522                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2523                /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2524                tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2525                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2526                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2527                           TILE_SPLIT(split_equal_to_row_size) |
2528                           NUM_BANKS(ADDR_SURF_16_BANK) |
2529                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2530                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2531                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2532                /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2533                tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2534                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2535                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2536                           TILE_SPLIT(split_equal_to_row_size) |
2537                           NUM_BANKS(ADDR_SURF_16_BANK) |
2538                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2539                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2540                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2541                /* 1D and 1D Array Surfaces */
2542                tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2543                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2544                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2545                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2546                           NUM_BANKS(ADDR_SURF_16_BANK) |
2547                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2548                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2549                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2550                /* Displayable maps. */
2551                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2552                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2553                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2554                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2555                           NUM_BANKS(ADDR_SURF_16_BANK) |
2556                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2557                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2558                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2559                /* Display 8bpp. */
2560                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2561                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2562                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2563                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2564                           NUM_BANKS(ADDR_SURF_16_BANK) |
2565                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2566                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2567                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2568                /* Display 16bpp. */
2569                tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2570                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2571                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2572                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2573                           NUM_BANKS(ADDR_SURF_16_BANK) |
2574                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2575                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2576                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2577                /* Display 32bpp. */
2578                tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2579                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2580                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2581                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2582                           NUM_BANKS(ADDR_SURF_16_BANK) |
2583                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2584                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2585                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2586                /* Thin. */
2587                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2588                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2589                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2590                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2591                           NUM_BANKS(ADDR_SURF_16_BANK) |
2592                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2593                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2594                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2595                /* Thin 8 bpp. */
2596                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2597                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2598                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2599                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2600                           NUM_BANKS(ADDR_SURF_16_BANK) |
2601                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2602                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2603                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2604                /* Thin 16 bpp. */
2605                tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2607                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2608                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2609                           NUM_BANKS(ADDR_SURF_16_BANK) |
2610                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2611                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2612                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2613                /* Thin 32 bpp. */
2614                tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2615                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2616                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2617                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2618                           NUM_BANKS(ADDR_SURF_16_BANK) |
2619                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2621                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2622                /* Thin 64 bpp. */
2623                tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2624                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2625                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2626                           TILE_SPLIT(split_equal_to_row_size) |
2627                           NUM_BANKS(ADDR_SURF_16_BANK) |
2628                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2629                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2630                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2631                /* 8 bpp PRT. */
2632                tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2633                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2634                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2635                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2636                           NUM_BANKS(ADDR_SURF_16_BANK) |
2637                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2638                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2639                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2640                /* 16 bpp PRT */
2641                tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2643                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2644                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2645                           NUM_BANKS(ADDR_SURF_16_BANK) |
2646                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2647                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2648                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2649                /* 32 bpp PRT */
2650                tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2651                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2652                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2653                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2654                           NUM_BANKS(ADDR_SURF_16_BANK) |
2655                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2656                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2657                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2658                /* 64 bpp PRT */
2659                tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2660                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2661                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2662                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2663                           NUM_BANKS(ADDR_SURF_16_BANK) |
2664                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2665                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2666                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2667                /* 128 bpp PRT */
2668                tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2669                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2670                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2671                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2672                           NUM_BANKS(ADDR_SURF_8_BANK) |
2673                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2674                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2675                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2676
2677                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2678                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2679                break;
2680
2681        case CHIP_VERDE:
2682        case CHIP_OLAND:
2683        case CHIP_HAINAN:
2684                /* non-AA compressed depth or any compressed stencil */
2685                tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2686                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2687                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2688                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2689                           NUM_BANKS(ADDR_SURF_16_BANK) |
2690                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2692                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2693                /* 2xAA/4xAA compressed depth only */
2694                tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2695                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2696                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2697                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2698                           NUM_BANKS(ADDR_SURF_16_BANK) |
2699                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2700                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2701                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2702                /* 8xAA compressed depth only */
2703                tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2704                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2705                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2706                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2707                           NUM_BANKS(ADDR_SURF_16_BANK) |
2708                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2710                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2711                /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2712                tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2713                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2714                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2715                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2716                           NUM_BANKS(ADDR_SURF_16_BANK) |
2717                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2718                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2719                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2720                /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2721                tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2722                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2723                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2724                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2725                           NUM_BANKS(ADDR_SURF_16_BANK) |
2726                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2728                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2729                /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2730                tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2731                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2732                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2733                           TILE_SPLIT(split_equal_to_row_size) |
2734                           NUM_BANKS(ADDR_SURF_16_BANK) |
2735                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2736                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2737                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2738                /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2739                tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2740                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2741                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2742                           TILE_SPLIT(split_equal_to_row_size) |
2743                           NUM_BANKS(ADDR_SURF_16_BANK) |
2744                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2746                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2747                /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2748                tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2749                           MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2750                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2751                           TILE_SPLIT(split_equal_to_row_size) |
2752                           NUM_BANKS(ADDR_SURF_16_BANK) |
2753                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2754                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2755                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2756                /* 1D and 1D Array Surfaces */
2757                tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2758                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2759                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2760                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2761                           NUM_BANKS(ADDR_SURF_16_BANK) |
2762                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2763                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2764                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2765                /* Displayable maps. */
2766                tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2767                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2768                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2769                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2770                           NUM_BANKS(ADDR_SURF_16_BANK) |
2771                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2772                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2773                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2774                /* Display 8bpp. */
2775                tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2776                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2777                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2778                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2779                           NUM_BANKS(ADDR_SURF_16_BANK) |
2780                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2781                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2782                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2783                /* Display 16bpp. */
2784                tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2786                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2787                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2788                           NUM_BANKS(ADDR_SURF_16_BANK) |
2789                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2790                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2791                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2792                /* Display 32bpp. */
2793                tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2794                           MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2795                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2796                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2797                           NUM_BANKS(ADDR_SURF_16_BANK) |
2798                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2799                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2800                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2801                /* Thin. */
2802                tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2803                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2804                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2805                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2806                           NUM_BANKS(ADDR_SURF_16_BANK) |
2807                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2808                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2809                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2810                /* Thin 8 bpp. */
2811                tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2812                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2813                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2815                           NUM_BANKS(ADDR_SURF_16_BANK) |
2816                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2817                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2818                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2819                /* Thin 16 bpp. */
2820                tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2821                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2822                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2823                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2824                           NUM_BANKS(ADDR_SURF_16_BANK) |
2825                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2826                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2827                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2828                /* Thin 32 bpp. */
2829                tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2830                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2831                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2832                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2833                           NUM_BANKS(ADDR_SURF_16_BANK) |
2834                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2835                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2836                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2837                /* Thin 64 bpp. */
2838                tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2839                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2840                           PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2841                           TILE_SPLIT(split_equal_to_row_size) |
2842                           NUM_BANKS(ADDR_SURF_16_BANK) |
2843                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2844                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2845                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2846                /* 8 bpp PRT. */
2847                tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2848                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2849                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2850                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2851                           NUM_BANKS(ADDR_SURF_16_BANK) |
2852                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2853                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2854                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2855                /* 16 bpp PRT */
2856                tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2857                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2858                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2859                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2860                           NUM_BANKS(ADDR_SURF_16_BANK) |
2861                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2862                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2863                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2864                /* 32 bpp PRT */
2865                tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2866                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2867                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2868                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2869                           NUM_BANKS(ADDR_SURF_16_BANK) |
2870                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2871                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2872                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2873                /* 64 bpp PRT */
2874                tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2875                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2876                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2877                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2878                           NUM_BANKS(ADDR_SURF_16_BANK) |
2879                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2880                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2881                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2882                /* 128 bpp PRT */
2883                tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2884                           MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2885                           PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2886                           TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2887                           NUM_BANKS(ADDR_SURF_8_BANK) |
2888                           BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2889                           BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2890                           MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2891
2892                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2893                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2894                break;
2895
2896        default:
2897                DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2898        }
2899}
2900
2901static void si_select_se_sh(struct radeon_device *rdev,
2902                            u32 se_num, u32 sh_num)
2903{
2904        u32 data = INSTANCE_BROADCAST_WRITES;
2905
2906        if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2907                data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2908        else if (se_num == 0xffffffff)
2909                data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2910        else if (sh_num == 0xffffffff)
2911                data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2912        else
2913                data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2914        WREG32(GRBM_GFX_INDEX, data);
2915}
2916
2917static u32 si_create_bitmask(u32 bit_width)
2918{
2919        u32 i, mask = 0;
2920
2921        for (i = 0; i < bit_width; i++) {
2922                mask <<= 1;
2923                mask |= 1;
2924        }
2925        return mask;
2926}
2927
2928static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2929{
2930        u32 data, mask;
2931
2932        data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2933        if (data & 1)
2934                data &= INACTIVE_CUS_MASK;
2935        else
2936                data = 0;
2937        data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2938
2939        data >>= INACTIVE_CUS_SHIFT;
2940
2941        mask = si_create_bitmask(cu_per_sh);
2942
2943        return ~data & mask;
2944}
2945
2946static void si_setup_spi(struct radeon_device *rdev,
2947                         u32 se_num, u32 sh_per_se,
2948                         u32 cu_per_sh)
2949{
2950        int i, j, k;
2951        u32 data, mask, active_cu;
2952
2953        for (i = 0; i < se_num; i++) {
2954                for (j = 0; j < sh_per_se; j++) {
2955                        si_select_se_sh(rdev, i, j);
2956                        data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2957                        active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2958
2959                        mask = 1;
2960                        for (k = 0; k < 16; k++) {
2961                                mask <<= k;
2962                                if (active_cu & mask) {
2963                                        data &= ~mask;
2964                                        WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2965                                        break;
2966                                }
2967                        }
2968                }
2969        }
2970        si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2971}
2972
2973static u32 si_get_rb_disabled(struct radeon_device *rdev,
2974                              u32 max_rb_num_per_se,
2975                              u32 sh_per_se)
2976{
2977        u32 data, mask;
2978
2979        data = RREG32(CC_RB_BACKEND_DISABLE);
2980        if (data & 1)
2981                data &= BACKEND_DISABLE_MASK;
2982        else
2983                data = 0;
2984        data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2985
2986        data >>= BACKEND_DISABLE_SHIFT;
2987
2988        mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
2989
2990        return data & mask;
2991}
2992
2993static void si_setup_rb(struct radeon_device *rdev,
2994                        u32 se_num, u32 sh_per_se,
2995                        u32 max_rb_num_per_se)
2996{
2997        int i, j;
2998        u32 data, mask;
2999        u32 disabled_rbs = 0;
3000        u32 enabled_rbs = 0;

3001
3002        for (i = 0; i < se_num; i++) {
3003                for (j = 0; j < sh_per_se; j++) {
3004                        si_select_se_sh(rdev, i, j);
3005                        data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3006                        disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3007                }
3008        }
3009        si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3010
3011        mask = 1;
3012        for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3013                if (!(disabled_rbs & mask))
3014                        enabled_rbs |= mask;
3015                mask <<= 1;
3016        }
3017
3018        rdev->config.si.backend_enable_mask = enabled_rbs;
3019
3020        for (i = 0; i < se_num; i++) {
3021                si_select_se_sh(rdev, i, 0xffffffff);
3022                data = 0;
3023                for (j = 0; j < sh_per_se; j++) {
3024                        switch (enabled_rbs & 3) {
3025                        case 1:
3026                                data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3027                                break;
3028                        case 2:
3029                                data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3030                                break;
3031                        case 3:
3032                        default:
3033                                data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3034                                break;
3035                        }
3036                        enabled_rbs >>= 2;
3037                }
3038                WREG32(PA_SC_RASTER_CONFIG, data);
3039        }
3040        si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3041}
3042
3043static void si_gpu_init(struct radeon_device *rdev)
3044{
3045        u32 gb_addr_config = 0;
3046        u32 mc_shared_chmap, mc_arb_ramcfg;
3047        u32 sx_debug_1;
3048        u32 hdp_host_path_cntl;
3049        u32 tmp;
3050        int i, j;
3051
3052        switch (rdev->family) {
3053        case CHIP_TAHITI:
3054                rdev->config.si.max_shader_engines = 2;
3055                rdev->config.si.max_tile_pipes = 12;
3056                rdev->config.si.max_cu_per_sh = 8;
3057                rdev->config.si.max_sh_per_se = 2;
3058                rdev->config.si.max_backends_per_se = 4;
3059                rdev->config.si.max_texture_channel_caches = 12;
3060                rdev->config.si.max_gprs = 256;
3061                rdev->config.si.max_gs_threads = 32;
3062                rdev->config.si.max_hw_contexts = 8;
3063
3064                rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3065                rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3066                rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3067                rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3068                gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3069                break;
3070        case CHIP_PITCAIRN:
3071                rdev->config.si.max_shader_engines = 2;
3072                rdev->config.si.max_tile_pipes = 8;
3073                rdev->config.si.max_cu_per_sh = 5;
3074                rdev->config.si.max_sh_per_se = 2;
3075                rdev->config.si.max_backends_per_se = 4;
3076                rdev->config.si.max_texture_channel_caches = 8;
3077                rdev->config.si.max_gprs = 256;
3078                rdev->config.si.max_gs_threads = 32;
3079                rdev->config.si.max_hw_contexts = 8;
3080
3081                rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3082                rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3083                rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3084                rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3085                gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3086                break;
3087        case CHIP_VERDE:
3088        default:
3089                rdev->config.si.max_shader_engines = 1;
3090                rdev->config.si.max_tile_pipes = 4;
3091                rdev->config.si.max_cu_per_sh = 5;
3092                rdev->config.si.max_sh_per_se = 2;
3093                rdev->config.si.max_backends_per_se = 4;
3094                rdev->config.si.max_texture_channel_caches = 4;
3095                rdev->config.si.max_gprs = 256;
3096                rdev->config.si.max_gs_threads = 32;
3097                rdev->config.si.max_hw_contexts = 8;
3098
3099                rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3100                rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3101                rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3102                rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3103                gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3104                break;
3105        case CHIP_OLAND:
3106                rdev->config.si.max_shader_engines = 1;
3107                rdev->config.si.max_tile_pipes = 4;
3108                rdev->config.si.max_cu_per_sh = 6;
3109                rdev->config.si.max_sh_per_se = 1;
3110                rdev->config.si.max_backends_per_se = 2;
3111                rdev->config.si.max_texture_channel_caches = 4;
3112                rdev->config.si.max_gprs = 256;
3113                rdev->config.si.max_gs_threads = 16;
3114                rdev->config.si.max_hw_contexts = 8;
3115
3116                rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3117                rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3118                rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3119                rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3120                gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3121                break;
3122        case CHIP_HAINAN:
3123                rdev->config.si.max_shader_engines = 1;
3124                rdev->config.si.max_tile_pipes = 4;
3125                rdev->config.si.max_cu_per_sh = 5;
3126                rdev->config.si.max_sh_per_se = 1;
3127                rdev->config.si.max_backends_per_se = 1;
3128                rdev->config.si.max_texture_channel_caches = 2;
3129                rdev->config.si.max_gprs = 256;
3130                rdev->config.si.max_gs_threads = 16;
3131                rdev->config.si.max_hw_contexts = 8;
3132
3133                rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3134                rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3135                rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3136                rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3137                gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3138                break;
3139        }
3140
3141        /* Initialize HDP */
3142        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3143                WREG32((0x2c14 + j), 0x00000000);
3144                WREG32((0x2c18 + j), 0x00000000);
3145                WREG32((0x2c1c + j), 0x00000000);
3146                WREG32((0x2c20 + j), 0x00000000);
3147                WREG32((0x2c24 + j), 0x00000000);
3148        }
3149
3150        WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3151        WREG32(SRBM_INT_CNTL, 1);
3152        WREG32(SRBM_INT_ACK, 1);
3153
3154        evergreen_fix_pci_max_read_req_size(rdev);
3155
3156        WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3157
3158        mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3159        mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3160
3161        rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3162        rdev->config.si.mem_max_burst_length_bytes = 256;
3163        tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3164        rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3165        if (rdev->config.si.mem_row_size_in_kb > 4)
3166                rdev->config.si.mem_row_size_in_kb = 4;
3167        /* XXX use MC settings? */
3168        rdev->config.si.shader_engine_tile_size = 32;
3169        rdev->config.si.num_gpus = 1;
3170        rdev->config.si.multi_gpu_tile_size = 64;
3171
3172        /* fix up row size */
3173        gb_addr_config &= ~ROW_SIZE_MASK;
3174        switch (rdev->config.si.mem_row_size_in_kb) {
3175        case 1:
3176        default:
3177                gb_addr_config |= ROW_SIZE(0);
3178                break;
3179        case 2:
3180                gb_addr_config |= ROW_SIZE(1);
3181                break;
3182        case 4:
3183                gb_addr_config |= ROW_SIZE(2);
3184                break;
3185        }
3186
3187        /* setup tiling info dword.  gb_addr_config is not adequate since it does
3188         * not have bank info, so create a custom tiling dword.
3189         * bits 3:0   num_pipes
3190         * bits 7:4   num_banks
3191         * bits 11:8  group_size
3192         * bits 15:12 row_size
3193         */
3194        rdev->config.si.tile_config = 0;
3195        switch (rdev->config.si.num_tile_pipes) {
3196        case 1:
3197                rdev->config.si.tile_config |= (0 << 0);
3198                break;
3199        case 2:
3200                rdev->config.si.tile_config |= (1 << 0);
3201                break;
3202        case 4:
3203                rdev->config.si.tile_config |= (2 << 0);
3204                break;
3205        case 8:
3206        default:
3207                /* XXX what about 12? */
3208                rdev->config.si.tile_config |= (3 << 0);
3209                break;
3210        }       
3211        switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3212        case 0: /* four banks */
3213                rdev->config.si.tile_config |= 0 << 4;
3214                break;
3215        case 1: /* eight banks */
3216                rdev->config.si.tile_config |= 1 << 4;
3217                break;
3218        case 2: /* sixteen banks */
3219        default:
3220                rdev->config.si.tile_config |= 2 << 4;
3221                break;
3222        }
3223        rdev->config.si.tile_config |=
3224                ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3225        rdev->config.si.tile_config |=
3226                ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3227
3228        WREG32(GB_ADDR_CONFIG, gb_addr_config);
3229        WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3230        WREG32(DMIF_ADDR_CALC, gb_addr_config);
3231        WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3232        WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3233        WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3234        if (rdev->has_uvd) {
3235                WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3236                WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3237                WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3238        }
3239
3240        si_tiling_mode_table_init(rdev);
3241
3242        si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3243                    rdev->config.si.max_sh_per_se,
3244                    rdev->config.si.max_backends_per_se);
3245
3246        si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3247                     rdev->config.si.max_sh_per_se,
3248                     rdev->config.si.max_cu_per_sh);
3249
3250        rdev->config.si.active_cus = 0;
3251        for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3252                for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3253                        rdev->config.si.active_cus +=
3254                                hweight32(si_get_cu_active_bitmap(rdev, i, j));
3255                }
3256        }
3257
3258        /* set HW defaults for 3D engine */
3259        WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3260                                     ROQ_IB2_START(0x2b)));
3261        WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3262
3263        sx_debug_1 = RREG32(SX_DEBUG_1);
3264        WREG32(SX_DEBUG_1, sx_debug_1);
3265
3266        WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3267
3268        WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3269                                 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3270                                 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3271                                 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3272
3273        WREG32(VGT_NUM_INSTANCES, 1);
3274
3275        WREG32(CP_PERFMON_CNTL, 0);
3276
3277        WREG32(SQ_CONFIG, 0);
3278
3279        WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3280                                          FORCE_EOV_MAX_REZ_CNT(255)));
3281
3282        WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3283               AUTO_INVLD_EN(ES_AND_GS_AUTO));
3284
3285        WREG32(VGT_GS_VERTEX_REUSE, 16);
3286        WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3287
3288        WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3289        WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3290        WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3291        WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3292        WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3293        WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3294        WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3295        WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3296
3297        tmp = RREG32(HDP_MISC_CNTL);
3298        tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3299        WREG32(HDP_MISC_CNTL, tmp);
3300
3301        hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3302        WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3303
3304        WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3305
3306        udelay(50);
3307}
3308
3309/*
3310 * GPU scratch registers helpers function.
3311 */
3312static void si_scratch_init(struct radeon_device *rdev)
3313{
3314        int i;
3315
3316        rdev->scratch.num_reg = 7;
3317        rdev->scratch.reg_base = SCRATCH_REG0;
3318        for (i = 0; i < rdev->scratch.num_reg; i++) {
3319                rdev->scratch.free[i] = true;
3320                rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3321        }
3322}
3323
3324void si_fence_ring_emit(struct radeon_device *rdev,
3325                        struct radeon_fence *fence)
3326{
3327        struct radeon_ring *ring = &rdev->ring[fence->ring];
3328        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3329
3330        /* flush read cache over gart */
3331        radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3332        radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3333        radeon_ring_write(ring, 0);
3334        radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3335        radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3336                          PACKET3_TC_ACTION_ENA |
3337                          PACKET3_SH_KCACHE_ACTION_ENA |
3338                          PACKET3_SH_ICACHE_ACTION_ENA);
3339        radeon_ring_write(ring, 0xFFFFFFFF);
3340        radeon_ring_write(ring, 0);
3341        radeon_ring_write(ring, 10); /* poll interval */
3342        /* EVENT_WRITE_EOP - flush caches, send int */
3343        radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3344        radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3345        radeon_ring_write(ring, lower_32_bits(addr));
3346        radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3347        radeon_ring_write(ring, fence->seq);
3348        radeon_ring_write(ring, 0);
3349}
3350
3351/*
3352 * IB stuff
3353 */
3354void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3355{
3356        struct radeon_ring *ring = &rdev->ring[ib->ring];
3357        unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3358        u32 header;
3359
3360        if (ib->is_const_ib) {
3361                /* set switch buffer packet before const IB */
3362                radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3363                radeon_ring_write(ring, 0);
3364
3365                header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3366        } else {
3367                u32 next_rptr;
3368                if (ring->rptr_save_reg) {
3369                        next_rptr = ring->wptr + 3 + 4 + 8;
3370                        radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3371                        radeon_ring_write(ring, ((ring->rptr_save_reg -
3372                                                  PACKET3_SET_CONFIG_REG_START) >> 2));
3373                        radeon_ring_write(ring, next_rptr);
3374                } else if (rdev->wb.enabled) {
3375                        next_rptr = ring->wptr + 5 + 4 + 8;
3376                        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3377                        radeon_ring_write(ring, (1 << 8));
3378                        radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3379                        radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3380                        radeon_ring_write(ring, next_rptr);
3381                }
3382
3383                header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3384        }
3385
3386        radeon_ring_write(ring, header);
3387        radeon_ring_write(ring,
3388#ifdef __BIG_ENDIAN
3389                          (2 << 0) |
3390#endif
3391                          (ib->gpu_addr & 0xFFFFFFFC));
3392        radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3393        radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
3394
3395        if (!ib->is_const_ib) {
3396                /* flush read cache over gart for this vmid */
3397                radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3398                radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3399                radeon_ring_write(ring, vm_id);
3400                radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3401                radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3402                                  PACKET3_TC_ACTION_ENA |
3403                                  PACKET3_SH_KCACHE_ACTION_ENA |
3404                                  PACKET3_SH_ICACHE_ACTION_ENA);
3405                radeon_ring_write(ring, 0xFFFFFFFF);
3406                radeon_ring_write(ring, 0);
3407                radeon_ring_write(ring, 10); /* poll interval */
3408        }
3409}
3410
3411/*
3412 * CP.
3413 */
3414static void si_cp_enable(struct radeon_device *rdev, bool enable)
3415{
3416        if (enable)
3417                WREG32(CP_ME_CNTL, 0);
3418        else {
3419                if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3420                        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3421                WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3422                WREG32(SCRATCH_UMSK, 0);
3423                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3424                rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3425                rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3426        }
3427        udelay(50);
3428}
3429
3430static int si_cp_load_microcode(struct radeon_device *rdev)
3431{
3432        int i;
3433
3434        if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3435                return -EINVAL;
3436
3437        si_cp_enable(rdev, false);
3438
3439        if (rdev->new_fw) {
3440                const struct gfx_firmware_header_v1_0 *pfp_hdr =
3441                        (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3442                const struct gfx_firmware_header_v1_0 *ce_hdr =
3443                        (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3444                const struct gfx_firmware_header_v1_0 *me_hdr =
3445                        (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3446                const __le32 *fw_data;
3447                u32 fw_size;
3448
3449                radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3450                radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3451                radeon_ucode_print_gfx_hdr(&me_hdr->header);
3452
3453                /* PFP */
3454                fw_data = (const __le32 *)
3455                        (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3456                fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3457                WREG32(CP_PFP_UCODE_ADDR, 0);
3458                for (i = 0; i < fw_size; i++)
3459                        WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3460                WREG32(CP_PFP_UCODE_ADDR, 0);
3461
3462                /* CE */
3463                fw_data = (const __le32 *)
3464                        (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3465                fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3466                WREG32(CP_CE_UCODE_ADDR, 0);
3467                for (i = 0; i < fw_size; i++)
3468                        WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3469                WREG32(CP_CE_UCODE_ADDR, 0);
3470
3471                /* ME */
3472                fw_data = (const __be32 *)
3473                        (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3474                fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3475                WREG32(CP_ME_RAM_WADDR, 0);
3476                for (i = 0; i < fw_size; i++)
3477                        WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3478                WREG32(CP_ME_RAM_WADDR, 0);
3479        } else {
3480                const __be32 *fw_data;
3481
3482                /* PFP */
3483                fw_data = (const __be32 *)rdev->pfp_fw->data;
3484                WREG32(CP_PFP_UCODE_ADDR, 0);
3485                for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3486                        WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3487                WREG32(CP_PFP_UCODE_ADDR, 0);
3488
3489                /* CE */
3490                fw_data = (const __be32 *)rdev->ce_fw->data;
3491                WREG32(CP_CE_UCODE_ADDR, 0);
3492                for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3493                        WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3494                WREG32(CP_CE_UCODE_ADDR, 0);
3495
3496                /* ME */
3497                fw_data = (const __be32 *)rdev->me_fw->data;
3498                WREG32(CP_ME_RAM_WADDR, 0);
3499                for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3500                        WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3501                WREG32(CP_ME_RAM_WADDR, 0);
3502        }
3503
3504        WREG32(CP_PFP_UCODE_ADDR, 0);
3505        WREG32(CP_CE_UCODE_ADDR, 0);
3506        WREG32(CP_ME_RAM_WADDR, 0);
3507        WREG32(CP_ME_RAM_RADDR, 0);
3508        return 0;
3509}
3510
3511static int si_cp_start(struct radeon_device *rdev)
3512{
3513        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3514        int r, i;
3515
3516        r = radeon_ring_lock(rdev, ring, 7 + 4);
3517        if (r) {
3518                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3519                return r;
3520        }
3521        /* init the CP */
3522        radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3523        radeon_ring_write(ring, 0x1);
3524        radeon_ring_write(ring, 0x0);
3525        radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3526        radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3527        radeon_ring_write(ring, 0);
3528        radeon_ring_write(ring, 0);
3529
3530        /* init the CE partitions */
3531        radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3532        radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3533        radeon_ring_write(ring, 0xc000);
3534        radeon_ring_write(ring, 0xe000);
3535        radeon_ring_unlock_commit(rdev, ring, false);
3536
3537        si_cp_enable(rdev, true);
3538
3539        r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3540        if (r) {
3541                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3542                return r;
3543        }
3544
3545        /* setup clear context state */
3546        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3547        radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3548
3549        for (i = 0; i < si_default_size; i++)
3550                radeon_ring_write(ring, si_default_state[i]);
3551
3552        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3553        radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3554
3555        /* set clear context state */
3556        radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3557        radeon_ring_write(ring, 0);
3558
3559        radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3560        radeon_ring_write(ring, 0x00000316);
3561        radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3562        radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3563
3564        radeon_ring_unlock_commit(rdev, ring, false);
3565
3566        for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3567                ring = &rdev->ring[i];
3568                r = radeon_ring_lock(rdev, ring, 2);
3569
3570                /* clear the compute context state */
3571                radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3572                radeon_ring_write(ring, 0);
3573
3574                radeon_ring_unlock_commit(rdev, ring, false);
3575        }
3576
3577        return 0;
3578}
3579
3580static void si_cp_fini(struct radeon_device *rdev)
3581{
3582        struct radeon_ring *ring;
3583        si_cp_enable(rdev, false);
3584
3585        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3586        radeon_ring_fini(rdev, ring);
3587        radeon_scratch_free(rdev, ring->rptr_save_reg);
3588
3589        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3590        radeon_ring_fini(rdev, ring);
3591        radeon_scratch_free(rdev, ring->rptr_save_reg);
3592
3593        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3594        radeon_ring_fini(rdev, ring);
3595        radeon_scratch_free(rdev, ring->rptr_save_reg);
3596}
3597
3598static int si_cp_resume(struct radeon_device *rdev)
3599{
3600        struct radeon_ring *ring;
3601        u32 tmp;
3602        u32 rb_bufsz;
3603        int r;
3604
3605        si_enable_gui_idle_interrupt(rdev, false);
3606
3607        WREG32(CP_SEM_WAIT_TIMER, 0x0);
3608        WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3609
3610        /* Set the write pointer delay */
3611        WREG32(CP_RB_WPTR_DELAY, 0);
3612
3613        WREG32(CP_DEBUG, 0);
3614        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3615
3616        /* ring 0 - compute and gfx */
3617        /* Set ring buffer size */
3618        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3619        rb_bufsz = order_base_2(ring->ring_size / 8);
3620        tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3621#ifdef __BIG_ENDIAN
3622        tmp |= BUF_SWAP_32BIT;
3623#endif
3624        WREG32(CP_RB0_CNTL, tmp);
3625
3626        /* Initialize the ring buffer's read and write pointers */
3627        WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3628        ring->wptr = 0;
3629        WREG32(CP_RB0_WPTR, ring->wptr);
3630
3631        /* set the wb address whether it's enabled or not */
3632        WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3633        WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3634
3635        if (rdev->wb.enabled)
3636                WREG32(SCRATCH_UMSK, 0xff);
3637        else {
3638                tmp |= RB_NO_UPDATE;
3639                WREG32(SCRATCH_UMSK, 0);
3640        }
3641
3642        mdelay(1);
3643        WREG32(CP_RB0_CNTL, tmp);
3644
3645        WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3646
3647        /* ring1  - compute only */
3648        /* Set ring buffer size */
3649        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3650        rb_bufsz = order_base_2(ring->ring_size / 8);
3651        tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3652#ifdef __BIG_ENDIAN
3653        tmp |= BUF_SWAP_32BIT;
3654#endif
3655        WREG32(CP_RB1_CNTL, tmp);
3656
3657        /* Initialize the ring buffer's read and write pointers */
3658        WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3659        ring->wptr = 0;
3660        WREG32(CP_RB1_WPTR, ring->wptr);
3661
3662        /* set the wb address whether it's enabled or not */
3663        WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3664        WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3665
3666        mdelay(1);
3667        WREG32(CP_RB1_CNTL, tmp);
3668
3669        WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3670
3671        /* ring2 - compute only */
3672        /* Set ring buffer size */
3673        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3674        rb_bufsz = order_base_2(ring->ring_size / 8);
3675        tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3676#ifdef __BIG_ENDIAN
3677        tmp |= BUF_SWAP_32BIT;
3678#endif
3679        WREG32(CP_RB2_CNTL, tmp);
3680
3681        /* Initialize the ring buffer's read and write pointers */
3682        WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3683        ring->wptr = 0;
3684        WREG32(CP_RB2_WPTR, ring->wptr);
3685
3686        /* set the wb address whether it's enabled or not */
3687        WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3688        WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3689
3690        mdelay(1);
3691        WREG32(CP_RB2_CNTL, tmp);
3692
3693        WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3694
3695        /* start the rings */
3696        si_cp_start(rdev);
3697        rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3698        rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3699        rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3700        r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3701        if (r) {
3702                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3703                rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3704                rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3705                return r;
3706        }
3707        r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3708        if (r) {
3709                rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3710        }
3711        r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3712        if (r) {
3713                rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3714        }
3715
3716        si_enable_gui_idle_interrupt(rdev, true);
3717
3718        if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3719                radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3720
3721        return 0;
3722}
3723
3724u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3725{
3726        u32 reset_mask = 0;
3727        u32 tmp;
3728
3729        /* GRBM_STATUS */
3730        tmp = RREG32(GRBM_STATUS);
3731        if (tmp & (PA_BUSY | SC_BUSY |
3732                   BCI_BUSY | SX_BUSY |
3733                   TA_BUSY | VGT_BUSY |
3734                   DB_BUSY | CB_BUSY |
3735                   GDS_BUSY | SPI_BUSY |
3736                   IA_BUSY | IA_BUSY_NO_DMA))
3737                reset_mask |= RADEON_RESET_GFX;
3738
3739        if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3740                   CP_BUSY | CP_COHERENCY_BUSY))
3741                reset_mask |= RADEON_RESET_CP;
3742
3743        if (tmp & GRBM_EE_BUSY)
3744                reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3745
3746        /* GRBM_STATUS2 */
3747        tmp = RREG32(GRBM_STATUS2);
3748        if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3749                reset_mask |= RADEON_RESET_RLC;
3750
3751        /* DMA_STATUS_REG 0 */
3752        tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3753        if (!(tmp & DMA_IDLE))
3754                reset_mask |= RADEON_RESET_DMA;
3755
3756        /* DMA_STATUS_REG 1 */
3757        tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3758        if (!(tmp & DMA_IDLE))
3759                reset_mask |= RADEON_RESET_DMA1;
3760
3761        /* SRBM_STATUS2 */
3762        tmp = RREG32(SRBM_STATUS2);
3763        if (tmp & DMA_BUSY)
3764                reset_mask |= RADEON_RESET_DMA;
3765
3766        if (tmp & DMA1_BUSY)
3767                reset_mask |= RADEON_RESET_DMA1;
3768
3769        /* SRBM_STATUS */
3770        tmp = RREG32(SRBM_STATUS);
3771
3772        if (tmp & IH_BUSY)
3773                reset_mask |= RADEON_RESET_IH;
3774
3775        if (tmp & SEM_BUSY)
3776                reset_mask |= RADEON_RESET_SEM;
3777
3778        if (tmp & GRBM_RQ_PENDING)
3779                reset_mask |= RADEON_RESET_GRBM;
3780
3781        if (tmp & VMC_BUSY)
3782                reset_mask |= RADEON_RESET_VMC;
3783
3784        if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3785                   MCC_BUSY | MCD_BUSY))
3786                reset_mask |= RADEON_RESET_MC;
3787
3788        if (evergreen_is_display_hung(rdev))
3789                reset_mask |= RADEON_RESET_DISPLAY;
3790
3791        /* VM_L2_STATUS */
3792        tmp = RREG32(VM_L2_STATUS);
3793        if (tmp & L2_BUSY)
3794                reset_mask |= RADEON_RESET_VMC;
3795
3796        /* Skip MC reset as it's mostly likely not hung, just busy */
3797        if (reset_mask & RADEON_RESET_MC) {
3798                DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3799                reset_mask &= ~RADEON_RESET_MC;
3800        }
3801
3802        return reset_mask;
3803}
3804
3805static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3806{
3807        struct evergreen_mc_save save;
3808        u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3809        u32 tmp;
3810
3811        if (reset_mask == 0)
3812                return;
3813
3814        dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3815
3816        evergreen_print_gpu_status_regs(rdev);
3817        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3818                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3819        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3820                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3821
3822        /* disable PG/CG */
3823        si_fini_pg(rdev);
3824        si_fini_cg(rdev);
3825
3826        /* stop the rlc */
3827        si_rlc_stop(rdev);
3828
3829        /* Disable CP parsing/prefetching */
3830        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3831
3832        if (reset_mask & RADEON_RESET_DMA) {
3833                /* dma0 */
3834                tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3835                tmp &= ~DMA_RB_ENABLE;
3836                WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3837        }
3838        if (reset_mask & RADEON_RESET_DMA1) {
3839                /* dma1 */
3840                tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3841                tmp &= ~DMA_RB_ENABLE;
3842                WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3843        }
3844
3845        udelay(50);
3846
3847        evergreen_mc_stop(rdev, &save);
3848        if (evergreen_mc_wait_for_idle(rdev)) {
3849                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3850        }
3851
3852        if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3853                grbm_soft_reset = SOFT_RESET_CB |
3854                        SOFT_RESET_DB |
3855                        SOFT_RESET_GDS |
3856                        SOFT_RESET_PA |
3857                        SOFT_RESET_SC |
3858                        SOFT_RESET_BCI |
3859                        SOFT_RESET_SPI |
3860                        SOFT_RESET_SX |
3861                        SOFT_RESET_TC |
3862                        SOFT_RESET_TA |
3863                        SOFT_RESET_VGT |
3864                        SOFT_RESET_IA;
3865        }
3866
3867        if (reset_mask & RADEON_RESET_CP) {
3868                grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3869
3870                srbm_soft_reset |= SOFT_RESET_GRBM;
3871        }
3872
3873        if (reset_mask & RADEON_RESET_DMA)
3874                srbm_soft_reset |= SOFT_RESET_DMA;
3875
3876        if (reset_mask & RADEON_RESET_DMA1)
3877                srbm_soft_reset |= SOFT_RESET_DMA1;
3878
3879        if (reset_mask & RADEON_RESET_DISPLAY)
3880                srbm_soft_reset |= SOFT_RESET_DC;
3881
3882        if (reset_mask & RADEON_RESET_RLC)
3883                grbm_soft_reset |= SOFT_RESET_RLC;
3884
3885        if (reset_mask & RADEON_RESET_SEM)
3886                srbm_soft_reset |= SOFT_RESET_SEM;
3887
3888        if (reset_mask & RADEON_RESET_IH)
3889                srbm_soft_reset |= SOFT_RESET_IH;
3890
3891        if (reset_mask & RADEON_RESET_GRBM)
3892                srbm_soft_reset |= SOFT_RESET_GRBM;
3893
3894        if (reset_mask & RADEON_RESET_VMC)
3895                srbm_soft_reset |= SOFT_RESET_VMC;
3896
3897        if (reset_mask & RADEON_RESET_MC)
3898                srbm_soft_reset |= SOFT_RESET_MC;
3899
3900        if (grbm_soft_reset) {
3901                tmp = RREG32(GRBM_SOFT_RESET);
3902                tmp |= grbm_soft_reset;
3903                dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3904                WREG32(GRBM_SOFT_RESET, tmp);
3905                tmp = RREG32(GRBM_SOFT_RESET);
3906
3907                udelay(50);
3908
3909                tmp &= ~grbm_soft_reset;
3910                WREG32(GRBM_SOFT_RESET, tmp);
3911                tmp = RREG32(GRBM_SOFT_RESET);
3912        }
3913
3914        if (srbm_soft_reset) {
3915                tmp = RREG32(SRBM_SOFT_RESET);
3916                tmp |= srbm_soft_reset;
3917                dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3918                WREG32(SRBM_SOFT_RESET, tmp);
3919                tmp = RREG32(SRBM_SOFT_RESET);
3920
3921                udelay(50);
3922
3923                tmp &= ~srbm_soft_reset;
3924                WREG32(SRBM_SOFT_RESET, tmp);
3925                tmp = RREG32(SRBM_SOFT_RESET);
3926        }
3927
3928        /* Wait a little for things to settle down */
3929        udelay(50);
3930
3931        evergreen_mc_resume(rdev, &save);
3932        udelay(50);
3933
3934        evergreen_print_gpu_status_regs(rdev);
3935}
3936
3937static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3938{
3939        u32 tmp, i;
3940
3941        tmp = RREG32(CG_SPLL_FUNC_CNTL);
3942        tmp |= SPLL_BYPASS_EN;
3943        WREG32(CG_SPLL_FUNC_CNTL, tmp);
3944
3945        tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3946        tmp |= SPLL_CTLREQ_CHG;
3947        WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3948
3949        for (i = 0; i < rdev->usec_timeout; i++) {
3950                if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3951                        break;
3952                udelay(1);
3953        }
3954
3955        tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3956        tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3957        WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3958
3959        tmp = RREG32(MPLL_CNTL_MODE);
3960        tmp &= ~MPLL_MCLK_SEL;
3961        WREG32(MPLL_CNTL_MODE, tmp);
3962}
3963
3964static void si_spll_powerdown(struct radeon_device *rdev)
3965{
3966        u32 tmp;
3967
3968        tmp = RREG32(SPLL_CNTL_MODE);
3969        tmp |= SPLL_SW_DIR_CONTROL;
3970        WREG32(SPLL_CNTL_MODE, tmp);
3971
3972        tmp = RREG32(CG_SPLL_FUNC_CNTL);
3973        tmp |= SPLL_RESET;
3974        WREG32(CG_SPLL_FUNC_CNTL, tmp);
3975
3976        tmp = RREG32(CG_SPLL_FUNC_CNTL);
3977        tmp |= SPLL_SLEEP;
3978        WREG32(CG_SPLL_FUNC_CNTL, tmp);
3979
3980        tmp = RREG32(SPLL_CNTL_MODE);
3981        tmp &= ~SPLL_SW_DIR_CONTROL;
3982        WREG32(SPLL_CNTL_MODE, tmp);
3983}
3984
3985static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3986{
3987        struct evergreen_mc_save save;
3988        u32 tmp, i;
3989
3990        dev_info(rdev->dev, "GPU pci config reset\n");
3991
3992        /* disable dpm? */
3993
3994        /* disable cg/pg */
3995        si_fini_pg(rdev);
3996        si_fini_cg(rdev);
3997
3998        /* Disable CP parsing/prefetching */
3999        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4000        /* dma0 */

4001        tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4002        tmp &= ~DMA_RB_ENABLE;
4003        WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4004        /* dma1 */
4005        tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4006        tmp &= ~DMA_RB_ENABLE;
4007        WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4008        /* XXX other engines? */
4009
4010        /* halt the rlc, disable cp internal ints */
4011        si_rlc_stop(rdev);
4012
4013        udelay(50);
4014
4015        /* disable mem access */
4016        evergreen_mc_stop(rdev, &save);
4017        if (evergreen_mc_wait_for_idle(rdev)) {
4018                dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4019        }
4020
4021        /* set mclk/sclk to bypass */
4022        si_set_clk_bypass_mode(rdev);
4023        /* powerdown spll */
4024        si_spll_powerdown(rdev);
4025        /* disable BM */
4026        pci_clear_master(rdev->pdev);
4027        /* reset */
4028        radeon_pci_config_reset(rdev);
4029        /* wait for asic to come out of reset */
4030        for (i = 0; i < rdev->usec_timeout; i++) {
4031                if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4032                        break;
4033                udelay(1);
4034        }
4035}
4036
4037int si_asic_reset(struct radeon_device *rdev, bool hard)
4038{
4039        u32 reset_mask;
4040
4041        if (hard) {
4042                si_gpu_pci_config_reset(rdev);
4043                return 0;
4044        }
4045
4046        reset_mask = si_gpu_check_soft_reset(rdev);
4047
4048        if (reset_mask)
4049                r600_set_bios_scratch_engine_hung(rdev, true);
4050
4051        /* try soft reset */
4052        si_gpu_soft_reset(rdev, reset_mask);
4053
4054        reset_mask = si_gpu_check_soft_reset(rdev);
4055
4056        /* try pci config reset */
4057        if (reset_mask && radeon_hard_reset)
4058                si_gpu_pci_config_reset(rdev);
4059
4060        reset_mask = si_gpu_check_soft_reset(rdev);
4061
4062        if (!reset_mask)
4063                r600_set_bios_scratch_engine_hung(rdev, false);
4064
4065        return 0;
4066}
4067
4068/**
4069 * si_gfx_is_lockup - Check if the GFX engine is locked up
4070 *
4071 * @rdev: radeon_device pointer
4072 * @ring: radeon_ring structure holding ring information
4073 *
4074 * Check if the GFX engine is locked up.
4075 * Returns true if the engine appears to be locked up, false if not.
4076 */
4077bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4078{
4079        u32 reset_mask = si_gpu_check_soft_reset(rdev);
4080
4081        if (!(reset_mask & (RADEON_RESET_GFX |
4082                            RADEON_RESET_COMPUTE |
4083                            RADEON_RESET_CP))) {
4084                radeon_ring_lockup_update(rdev, ring);
4085                return false;
4086        }
4087        return radeon_ring_test_lockup(rdev, ring);
4088}
4089
4090/* MC */
4091static void si_mc_program(struct radeon_device *rdev)
4092{
4093        struct evergreen_mc_save save;
4094        u32 tmp;
4095        int i, j;
4096
4097        /* Initialize HDP */
4098        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4099                WREG32((0x2c14 + j), 0x00000000);
4100                WREG32((0x2c18 + j), 0x00000000);
4101                WREG32((0x2c1c + j), 0x00000000);
4102                WREG32((0x2c20 + j), 0x00000000);
4103                WREG32((0x2c24 + j), 0x00000000);
4104        }
4105        WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4106
4107        evergreen_mc_stop(rdev, &save);
4108        if (radeon_mc_wait_for_idle(rdev)) {
4109                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4110        }
4111        if (!ASIC_IS_NODCE(rdev))
4112                /* Lockout access through VGA aperture*/
4113                WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4114        /* Update configuration */
4115        WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4116               rdev->mc.vram_start >> 12);
4117        WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4118               rdev->mc.vram_end >> 12);
4119        WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4120               rdev->vram_scratch.gpu_addr >> 12);
4121        tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4122        tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4123        WREG32(MC_VM_FB_LOCATION, tmp);
4124        /* XXX double check these! */
4125        WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4126        WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4127        WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4128        WREG32(MC_VM_AGP_BASE, 0);
4129        WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4130        WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4131        if (radeon_mc_wait_for_idle(rdev)) {
4132                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4133        }
4134        evergreen_mc_resume(rdev, &save);
4135        if (!ASIC_IS_NODCE(rdev)) {
4136                /* we need to own VRAM, so turn off the VGA renderer here
4137                 * to stop it overwriting our objects */
4138                rv515_vga_render_disable(rdev);
4139        }
4140}
4141
4142void si_vram_gtt_location(struct radeon_device *rdev,
4143                          struct radeon_mc *mc)
4144{
4145        if (mc->mc_vram_size > 0xFFC0000000ULL) {
4146                /* leave room for at least 1024M GTT */
4147                dev_warn(rdev->dev, "limiting VRAM\n");
4148                mc->real_vram_size = 0xFFC0000000ULL;
4149                mc->mc_vram_size = 0xFFC0000000ULL;
4150        }
4151        radeon_vram_location(rdev, &rdev->mc, 0);
4152        rdev->mc.gtt_base_align = 0;
4153        radeon_gtt_location(rdev, mc);
4154}
4155
4156static int si_mc_init(struct radeon_device *rdev)
4157{
4158        u32 tmp;
4159        int chansize, numchan;
4160
4161        /* Get VRAM informations */
4162        rdev->mc.vram_is_ddr = true;
4163        tmp = RREG32(MC_ARB_RAMCFG);
4164        if (tmp & CHANSIZE_OVERRIDE) {
4165                chansize = 16;
4166        } else if (tmp & CHANSIZE_MASK) {
4167                chansize = 64;
4168        } else {
4169                chansize = 32;
4170        }
4171        tmp = RREG32(MC_SHARED_CHMAP);
4172        switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4173        case 0:
4174        default:
4175                numchan = 1;
4176                break;
4177        case 1:
4178                numchan = 2;
4179                break;
4180        case 2:
4181                numchan = 4;
4182                break;
4183        case 3:
4184                numchan = 8;
4185                break;
4186        case 4:
4187                numchan = 3;
4188                break;
4189        case 5:
4190                numchan = 6;
4191                break;
4192        case 6:
4193                numchan = 10;
4194                break;
4195        case 7:
4196                numchan = 12;
4197                break;
4198        case 8:
4199                numchan = 16;
4200                break;
4201        }
4202        rdev->mc.vram_width = numchan * chansize;
4203        /* Could aper size report 0 ? */
4204        rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4205        rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4206        /* size in MB on si */
4207        tmp = RREG32(CONFIG_MEMSIZE);
4208        /* some boards may have garbage in the upper 16 bits */
4209        if (tmp & 0xffff0000) {
4210                DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4211                if (tmp & 0xffff)
4212                        tmp &= 0xffff;
4213        }
4214        rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4215        rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4216        rdev->mc.visible_vram_size = rdev->mc.aper_size;
4217        si_vram_gtt_location(rdev, &rdev->mc);
4218        radeon_update_bandwidth_info(rdev);
4219
4220        return 0;
4221}
4222
4223/*
4224 * GART
4225 */
4226void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4227{
4228        /* flush hdp cache */
4229        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4230
4231        /* bits 0-15 are the VM contexts0-15 */
4232        WREG32(VM_INVALIDATE_REQUEST, 1);
4233}
4234
4235static int si_pcie_gart_enable(struct radeon_device *rdev)
4236{
4237        int r, i;
4238
4239        if (rdev->gart.robj == NULL) {
4240                dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4241                return -EINVAL;
4242        }
4243        r = radeon_gart_table_vram_pin(rdev);
4244        if (r)
4245                return r;
4246        /* Setup TLB control */
4247        WREG32(MC_VM_MX_L1_TLB_CNTL,
4248               (0xA << 7) |
4249               ENABLE_L1_TLB |
4250               ENABLE_L1_FRAGMENT_PROCESSING |
4251               SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4252               ENABLE_ADVANCED_DRIVER_MODEL |
4253               SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4254        /* Setup L2 cache */
4255        WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4256               ENABLE_L2_FRAGMENT_PROCESSING |
4257               ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4258               ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4259               EFFECTIVE_L2_QUEUE_SIZE(7) |
4260               CONTEXT1_IDENTITY_ACCESS_MODE(1));
4261        WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4262        WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4263               BANK_SELECT(4) |
4264               L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4265        /* setup context0 */
4266        WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4267        WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4268        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4269        WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4270                        (u32)(rdev->dummy_page.addr >> 12));
4271        WREG32(VM_CONTEXT0_CNTL2, 0);
4272        WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4273                                  RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4274
4275        WREG32(0x15D4, 0);
4276        WREG32(0x15D8, 0);
4277        WREG32(0x15DC, 0);
4278
4279        /* empty context1-15 */
4280        /* set vm size, must be a multiple of 4 */
4281        WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4282        WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
4283        /* Assign the pt base to something valid for now; the pts used for
4284         * the VMs are determined by the application and setup and assigned
4285         * on the fly in the vm part of radeon_gart.c
4286         */
4287        for (i = 1; i < 16; i++) {
4288                if (i < 8)
4289                        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4290                               rdev->vm_manager.saved_table_addr[i]);
4291                else
4292                        WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4293                               rdev->vm_manager.saved_table_addr[i]);
4294        }
4295
4296        /* enable context1-15 */
4297        WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4298               (u32)(rdev->dummy_page.addr >> 12));
4299        WREG32(VM_CONTEXT1_CNTL2, 4);
4300        WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4301                                PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4302                                RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4303                                RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4304                                DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4305                                DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4306                                PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4307                                PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4308                                VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4309                                VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4310                                READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4311                                READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4312                                WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4313                                WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4314
4315        si_pcie_gart_tlb_flush(rdev);
4316        DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4317                 (unsigned)(rdev->mc.gtt_size >> 20),
4318                 (unsigned long long)rdev->gart.table_addr);
4319        rdev->gart.ready = true;
4320        return 0;
4321}
4322
4323static void si_pcie_gart_disable(struct radeon_device *rdev)
4324{
4325        unsigned i;
4326
4327        for (i = 1; i < 16; ++i) {
4328                uint32_t reg;
4329                if (i < 8)
4330                        reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4331                else
4332                        reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4333                rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4334        }
4335
4336        /* Disable all tables */
4337        WREG32(VM_CONTEXT0_CNTL, 0);
4338        WREG32(VM_CONTEXT1_CNTL, 0);
4339        /* Setup TLB control */
4340        WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4341               SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4342        /* Setup L2 cache */
4343        WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4344               ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4345               EFFECTIVE_L2_QUEUE_SIZE(7) |
4346               CONTEXT1_IDENTITY_ACCESS_MODE(1));
4347        WREG32(VM_L2_CNTL2, 0);
4348        WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4349               L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4350        radeon_gart_table_vram_unpin(rdev);
4351}
4352
4353static void si_pcie_gart_fini(struct radeon_device *rdev)
4354{
4355        si_pcie_gart_disable(rdev);
4356        radeon_gart_table_vram_free(rdev);
4357        radeon_gart_fini(rdev);
4358}
4359
4360/* vm parser */
4361static bool si_vm_reg_valid(u32 reg)
4362{
4363        /* context regs are fine */
4364        if (reg >= 0x28000)
4365                return true;
4366
4367        /* shader regs are also fine */
4368        if (reg >= 0xB000 && reg < 0xC000)
4369                return true;
4370
4371        /* check config regs */
4372        switch (reg) {
4373        case GRBM_GFX_INDEX:
4374        case CP_STRMOUT_CNTL:
4375        case VGT_VTX_VECT_EJECT_REG:
4376        case VGT_CACHE_INVALIDATION:
4377        case VGT_ESGS_RING_SIZE:
4378        case VGT_GSVS_RING_SIZE:
4379        case VGT_GS_VERTEX_REUSE:
4380        case VGT_PRIMITIVE_TYPE:
4381        case VGT_INDEX_TYPE:
4382        case VGT_NUM_INDICES:
4383        case VGT_NUM_INSTANCES:
4384        case VGT_TF_RING_SIZE:
4385        case VGT_HS_OFFCHIP_PARAM:
4386        case VGT_TF_MEMORY_BASE:
4387        case PA_CL_ENHANCE:
4388        case PA_SU_LINE_STIPPLE_VALUE:
4389        case PA_SC_LINE_STIPPLE_STATE:
4390        case PA_SC_ENHANCE:
4391        case SQC_CACHES:
4392        case SPI_STATIC_THREAD_MGMT_1:
4393        case SPI_STATIC_THREAD_MGMT_2:
4394        case SPI_STATIC_THREAD_MGMT_3:
4395        case SPI_PS_MAX_WAVE_ID:
4396        case SPI_CONFIG_CNTL:
4397        case SPI_CONFIG_CNTL_1:
4398        case TA_CNTL_AUX:
4399                return true;
4400        default:
4401                DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4402                return false;
4403        }
4404}
4405
4406static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4407                                  u32 *ib, struct radeon_cs_packet *pkt)
4408{
4409        switch (pkt->opcode) {
4410        case PACKET3_NOP:
4411        case PACKET3_SET_BASE:
4412        case PACKET3_SET_CE_DE_COUNTERS:
4413        case PACKET3_LOAD_CONST_RAM:
4414        case PACKET3_WRITE_CONST_RAM:
4415        case PACKET3_WRITE_CONST_RAM_OFFSET:
4416        case PACKET3_DUMP_CONST_RAM:
4417        case PACKET3_INCREMENT_CE_COUNTER:
4418        case PACKET3_WAIT_ON_DE_COUNTER:
4419        case PACKET3_CE_WRITE:
4420                break;
4421        default:
4422                DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4423                return -EINVAL;
4424        }
4425        return 0;
4426}
4427
4428static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4429{
4430        u32 start_reg, reg, i;
4431        u32 command = ib[idx + 4];
4432        u32 info = ib[idx + 1];
4433        u32 idx_value = ib[idx];
4434        if (command & PACKET3_CP_DMA_CMD_SAS) {
4435                /* src address space is register */
4436                if (((info & 0x60000000) >> 29) == 0) {
4437                        start_reg = idx_value << 2;
4438                        if (command & PACKET3_CP_DMA_CMD_SAIC) {
4439                                reg = start_reg;
4440                                if (!si_vm_reg_valid(reg)) {
4441                                        DRM_ERROR("CP DMA Bad SRC register\n");
4442                                        return -EINVAL;
4443                                }
4444                        } else {
4445                                for (i = 0; i < (command & 0x1fffff); i++) {
4446                                        reg = start_reg + (4 * i);
4447                                        if (!si_vm_reg_valid(reg)) {
4448                                                DRM_ERROR("CP DMA Bad SRC register\n");
4449                                                return -EINVAL;
4450                                        }
4451                                }
4452                        }
4453                }
4454        }
4455        if (command & PACKET3_CP_DMA_CMD_DAS) {
4456                /* dst address space is register */
4457                if (((info & 0x00300000) >> 20) == 0) {
4458                        start_reg = ib[idx + 2];
4459                        if (command & PACKET3_CP_DMA_CMD_DAIC) {
4460                                reg = start_reg;
4461                                if (!si_vm_reg_valid(reg)) {
4462                                        DRM_ERROR("CP DMA Bad DST register\n");
4463                                        return -EINVAL;
4464                                }
4465                        } else {
4466                                for (i = 0; i < (command & 0x1fffff); i++) {
4467                                        reg = start_reg + (4 * i);
4468                                if (!si_vm_reg_valid(reg)) {
4469                                                DRM_ERROR("CP DMA Bad DST register\n");
4470                                                return -EINVAL;
4471                                        }
4472                                }
4473                        }
4474                }
4475        }
4476        return 0;
4477}
4478
4479static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4480                                   u32 *ib, struct radeon_cs_packet *pkt)
4481{
4482        int r;
4483        u32 idx = pkt->idx + 1;
4484        u32 idx_value = ib[idx];
4485        u32 start_reg, end_reg, reg, i;
4486
4487        switch (pkt->opcode) {
4488        case PACKET3_NOP:
4489        case PACKET3_SET_BASE:
4490        case PACKET3_CLEAR_STATE:
4491        case PACKET3_INDEX_BUFFER_SIZE:
4492        case PACKET3_DISPATCH_DIRECT:
4493        case PACKET3_DISPATCH_INDIRECT:
4494        case PACKET3_ALLOC_GDS:
4495        case PACKET3_WRITE_GDS_RAM:
4496        case PACKET3_ATOMIC_GDS:
4497        case PACKET3_ATOMIC:
4498        case PACKET3_OCCLUSION_QUERY:
4499        case PACKET3_SET_PREDICATION:
4500        case PACKET3_COND_EXEC:
4501        case PACKET3_PRED_EXEC:
4502        case PACKET3_DRAW_INDIRECT:
4503        case PACKET3_DRAW_INDEX_INDIRECT:
4504        case PACKET3_INDEX_BASE:
4505        case PACKET3_DRAW_INDEX_2:
4506        case PACKET3_CONTEXT_CONTROL:
4507        case PACKET3_INDEX_TYPE:
4508        case PACKET3_DRAW_INDIRECT_MULTI:
4509        case PACKET3_DRAW_INDEX_AUTO:
4510        case PACKET3_DRAW_INDEX_IMMD:
4511        case PACKET3_NUM_INSTANCES:
4512        case PACKET3_DRAW_INDEX_MULTI_AUTO:
4513        case PACKET3_STRMOUT_BUFFER_UPDATE:
4514        case PACKET3_DRAW_INDEX_OFFSET_2:
4515        case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4516        case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4517        case PACKET3_MPEG_INDEX:
4518        case PACKET3_WAIT_REG_MEM:
4519        case PACKET3_MEM_WRITE:
4520        case PACKET3_PFP_SYNC_ME:
4521        case PACKET3_SURFACE_SYNC:
4522        case PACKET3_EVENT_WRITE:
4523        case PACKET3_EVENT_WRITE_EOP:
4524        case PACKET3_EVENT_WRITE_EOS:
4525        case PACKET3_SET_CONTEXT_REG:
4526        case PACKET3_SET_CONTEXT_REG_INDIRECT:
4527        case PACKET3_SET_SH_REG:
4528        case PACKET3_SET_SH_REG_OFFSET:
4529        case PACKET3_INCREMENT_DE_COUNTER:
4530        case PACKET3_WAIT_ON_CE_COUNTER:
4531        case PACKET3_WAIT_ON_AVAIL_BUFFER:
4532        case PACKET3_ME_WRITE:
4533                break;
4534        case PACKET3_COPY_DATA:
4535                if ((idx_value & 0xf00) == 0) {
4536                        reg = ib[idx + 3] * 4;
4537                        if (!si_vm_reg_valid(reg))
4538                                return -EINVAL;
4539                }
4540                break;
4541        case PACKET3_WRITE_DATA:
4542                if ((idx_value & 0xf00) == 0) {
4543                        start_reg = ib[idx + 1] * 4;
4544                        if (idx_value & 0x10000) {
4545                                if (!si_vm_reg_valid(start_reg))
4546                                        return -EINVAL;
4547                        } else {
4548                                for (i = 0; i < (pkt->count - 2); i++) {
4549                                        reg = start_reg + (4 * i);
4550                                        if (!si_vm_reg_valid(reg))
4551                                                return -EINVAL;
4552                                }
4553                        }
4554                }
4555                break;
4556        case PACKET3_COND_WRITE:
4557                if (idx_value & 0x100) {
4558                        reg = ib[idx + 5] * 4;
4559                        if (!si_vm_reg_valid(reg))
4560                                return -EINVAL;
4561                }
4562                break;
4563        case PACKET3_COPY_DW:
4564                if (idx_value & 0x2) {
4565                        reg = ib[idx + 3] * 4;
4566                        if (!si_vm_reg_valid(reg))
4567                                return -EINVAL;
4568                }
4569                break;
4570        case PACKET3_SET_CONFIG_REG:
4571                start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4572                end_reg = 4 * pkt->count + start_reg - 4;
4573                if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4574                    (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4575                    (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4576                        DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4577                        return -EINVAL;
4578                }
4579                for (i = 0; i < pkt->count; i++) {
4580                        reg = start_reg + (4 * i);
4581                        if (!si_vm_reg_valid(reg))
4582                                return -EINVAL;
4583                }
4584                break;
4585        case PACKET3_CP_DMA:
4586                r = si_vm_packet3_cp_dma_check(ib, idx);
4587                if (r)
4588                        return r;
4589                break;
4590        default:
4591                DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4592                return -EINVAL;
4593        }
4594        return 0;
4595}
4596
4597static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4598                                       u32 *ib, struct radeon_cs_packet *pkt)
4599{
4600        int r;
4601        u32 idx = pkt->idx + 1;
4602        u32 idx_value = ib[idx];
4603        u32 start_reg, reg, i;
4604
4605        switch (pkt->opcode) {
4606        case PACKET3_NOP:
4607        case PACKET3_SET_BASE:
4608        case PACKET3_CLEAR_STATE:
4609        case PACKET3_DISPATCH_DIRECT:
4610        case PACKET3_DISPATCH_INDIRECT:
4611        case PACKET3_ALLOC_GDS:
4612        case PACKET3_WRITE_GDS_RAM:
4613        case PACKET3_ATOMIC_GDS:
4614        case PACKET3_ATOMIC:
4615        case PACKET3_OCCLUSION_QUERY:
4616        case PACKET3_SET_PREDICATION:
4617        case PACKET3_COND_EXEC:
4618        case PACKET3_PRED_EXEC:
4619        case PACKET3_CONTEXT_CONTROL:
4620        case PACKET3_STRMOUT_BUFFER_UPDATE:
4621        case PACKET3_WAIT_REG_MEM:
4622        case PACKET3_MEM_WRITE:
4623        case PACKET3_PFP_SYNC_ME:
4624        case PACKET3_SURFACE_SYNC:
4625        case PACKET3_EVENT_WRITE:
4626        case PACKET3_EVENT_WRITE_EOP:
4627        case PACKET3_EVENT_WRITE_EOS:
4628        case PACKET3_SET_CONTEXT_REG:
4629        case PACKET3_SET_CONTEXT_REG_INDIRECT:
4630        case PACKET3_SET_SH_REG:
4631        case PACKET3_SET_SH_REG_OFFSET:
4632        case PACKET3_INCREMENT_DE_COUNTER:
4633        case PACKET3_WAIT_ON_CE_COUNTER:
4634        case PACKET3_WAIT_ON_AVAIL_BUFFER:
4635        case PACKET3_ME_WRITE:
4636                break;
4637        case PACKET3_COPY_DATA:
4638                if ((idx_value & 0xf00) == 0) {
4639                        reg = ib[idx + 3] * 4;
4640                        if (!si_vm_reg_valid(reg))
4641                                return -EINVAL;
4642                }
4643                break;
4644        case PACKET3_WRITE_DATA:
4645                if ((idx_value & 0xf00) == 0) {
4646                        start_reg = ib[idx + 1] * 4;
4647                        if (idx_value & 0x10000) {
4648                                if (!si_vm_reg_valid(start_reg))
4649                                        return -EINVAL;
4650                        } else {
4651                                for (i = 0; i < (pkt->count - 2); i++) {
4652                                        reg = start_reg + (4 * i);
4653                                        if (!si_vm_reg_valid(reg))
4654                                                return -EINVAL;
4655                                }
4656                        }
4657                }
4658                break;
4659        case PACKET3_COND_WRITE:
4660                if (idx_value & 0x100) {
4661                        reg = ib[idx + 5] * 4;
4662                        if (!si_vm_reg_valid(reg))
4663                                return -EINVAL;
4664                }
4665                break;
4666        case PACKET3_COPY_DW:
4667                if (idx_value & 0x2) {
4668                        reg = ib[idx + 3] * 4;
4669                        if (!si_vm_reg_valid(reg))
4670                                return -EINVAL;
4671                }
4672                break;
4673        case PACKET3_CP_DMA:
4674                r = si_vm_packet3_cp_dma_check(ib, idx);
4675                if (r)
4676                        return r;
4677                break;
4678        default:
4679                DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4680                return -EINVAL;
4681        }
4682        return 0;
4683}
4684
4685int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4686{
4687        int ret = 0;
4688        u32 idx = 0, i;
4689        struct radeon_cs_packet pkt;
4690
4691        do {
4692                pkt.idx = idx;
4693                pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4694                pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4695                pkt.one_reg_wr = 0;
4696                switch (pkt.type) {
4697                case RADEON_PACKET_TYPE0:
4698                        dev_err(rdev->dev, "Packet0 not allowed!\n");
4699                        ret = -EINVAL;
4700                        break;
4701                case RADEON_PACKET_TYPE2:
4702                        idx += 1;
4703                        break;
4704                case RADEON_PACKET_TYPE3:
4705                        pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4706                        if (ib->is_const_ib)
4707                                ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4708                        else {
4709                                switch (ib->ring) {
4710                                case RADEON_RING_TYPE_GFX_INDEX:
4711                                        ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4712                                        break;
4713                                case CAYMAN_RING_TYPE_CP1_INDEX:
4714                                case CAYMAN_RING_TYPE_CP2_INDEX:
4715                                        ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4716                                        break;
4717                                default:
4718                                        dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4719                                        ret = -EINVAL;
4720                                        break;
4721                                }
4722                        }
4723                        idx += pkt.count + 2;
4724                        break;
4725                default:
4726                        dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4727                        ret = -EINVAL;
4728                        break;
4729                }
4730                if (ret) {
4731                        for (i = 0; i < ib->length_dw; i++) {
4732                                if (i == idx)
4733                                        printk("\t0x%08x <---\n", ib->ptr[i]);
4734                                else
4735                                        printk("\t0x%08x\n", ib->ptr[i]);
4736                        }
4737                        break;
4738                }
4739        } while (idx < ib->length_dw);
4740
4741        return ret;
4742}
4743
4744/*
4745 * vm
4746 */
4747int si_vm_init(struct radeon_device *rdev)
4748{
4749        /* number of VMs */
4750        rdev->vm_manager.nvm = 16;
4751        /* base offset of vram pages */
4752        rdev->vm_manager.vram_base_offset = 0;
4753
4754        return 0;
4755}
4756
4757void si_vm_fini(struct radeon_device *rdev)
4758{
4759}
4760
4761/**
4762 * si_vm_decode_fault - print human readable fault info
4763 *
4764 * @rdev: radeon_device pointer
4765 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4766 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4767 *
4768 * Print human readable fault information (SI).
4769 */
4770static void si_vm_decode_fault(struct radeon_device *rdev,
4771                               u32 status, u32 addr)
4772{
4773        u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4774        u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4775        u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4776        char *block;
4777
4778        if (rdev->family == CHIP_TAHITI) {
4779                switch (mc_id) {
4780                case 160:
4781                case 144:
4782                case 96:
4783                case 80:
4784                case 224:
4785                case 208:
4786                case 32:
4787                case 16:
4788                        block = "CB";
4789                        break;
4790                case 161:
4791                case 145:
4792                case 97:
4793                case 81:
4794                case 225:
4795                case 209:
4796                case 33:
4797                case 17:
4798                        block = "CB_FMASK";
4799                        break;
4800                case 162:
4801                case 146:
4802                case 98:
4803                case 82:
4804                case 226:
4805                case 210:
4806                case 34:
4807                case 18:
4808                        block = "CB_CMASK";
4809                        break;
4810                case 163:
4811                case 147:
4812                case 99:
4813                case 83:
4814                case 227:
4815                case 211:
4816                case 35:
4817                case 19:
4818                        block = "CB_IMMED";
4819                        break;
4820                case 164:
4821                case 148:
4822                case 100:
4823                case 84:
4824                case 228:
4825                case 212:
4826                case 36:
4827                case 20:
4828                        block = "DB";
4829                        break;
4830                case 165:
4831                case 149:
4832                case 101:
4833                case 85:
4834                case 229:
4835                case 213:
4836                case 37:
4837                case 21:
4838                        block = "DB_HTILE";
4839                        break;
4840                case 167:
4841                case 151:
4842                case 103:
4843                case 87:
4844                case 231:
4845                case 215:
4846                case 39:
4847                case 23:
4848                        block = "DB_STEN";
4849                        break;
4850                case 72:
4851                case 68:
4852                case 64:
4853                case 8:
4854                case 4:
4855                case 0:
4856                case 136:
4857                case 132:
4858                case 128:
4859                case 200:
4860                case 196:
4861                case 192:
4862                        block = "TC";
4863                        break;
4864                case 112:
4865                case 48:
4866                        block = "CP";
4867                        break;
4868                case 49:
4869                case 177:
4870                case 50:
4871                case 178:
4872                        block = "SH";
4873                        break;
4874                case 53:
4875                case 190:
4876                        block = "VGT";
4877                        break;
4878                case 117:
4879                        block = "IH";
4880                        break;
4881                case 51:
4882                case 115:
4883                        block = "RLC";
4884                        break;
4885                case 119:
4886                case 183:
4887                        block = "DMA0";
4888                        break;
4889                case 61:
4890                        block = "DMA1";
4891                        break;
4892                case 248:
4893                case 120:
4894                        block = "HDP";
4895                        break;
4896                default:
4897                        block = "unknown";
4898                        break;
4899                }
4900        } else {
4901                switch (mc_id) {
4902                case 32:
4903                case 16:
4904                case 96:
4905                case 80:
4906                case 160:
4907                case 144:
4908                case 224:
4909                case 208:
4910                        block = "CB";
4911                        break;
4912                case 33:
4913                case 17:
4914                case 97:
4915                case 81:
4916                case 161:
4917                case 145:
4918                case 225:
4919                case 209:
4920                        block = "CB_FMASK";
4921                        break;
4922                case 34:
4923                case 18:
4924                case 98:
4925                case 82:
4926                case 162:
4927                case 146:
4928                case 226:
4929                case 210:
4930                        block = "CB_CMASK";
4931                        break;
4932                case 35:
4933                case 19:
4934                case 99:
4935                case 83:
4936                case 163:
4937                case 147:
4938                case 227:
4939                case 211:
4940                        block = "CB_IMMED";
4941                        break;
4942                case 36:
4943                case 20:
4944                case 100:
4945                case 84:
4946                case 164:
4947                case 148:
4948                case 228:
4949                case 212:
4950                        block = "DB";
4951                        break;
4952                case 37:
4953                case 21:
4954                case 101:
4955                case 85:
4956                case 165:
4957                case 149:
4958                case 229:
4959                case 213:
4960                        block = "DB_HTILE";
4961                        break;
4962                case 39:
4963                case 23:
4964                case 103:
4965                case 87:
4966                case 167:
4967                case 151:
4968                case 231:
4969                case 215:
4970                        block = "DB_STEN";
4971                        break;
4972                case 72:
4973                case 68:
4974                case 8:
4975                case 4:
4976                case 136:
4977                case 132:
4978                case 200:
4979                case 196:
4980                        block = "TC";
4981                        break;
4982                case 112:
4983                case 48:
4984                        block = "CP";
4985                        break;
4986                case 49:
4987                case 177:
4988                case 50:
4989                case 178:
4990                        block = "SH";
4991                        break;
4992                case 53:
4993                        block = "VGT";
4994                        break;
4995                case 117:
4996                        block = "IH";
4997                        break;
4998                case 51:
4999                case 115:
5000                        block = "RLC";

5001                        break;
5002                case 119:
5003                case 183:
5004                        block = "DMA0";
5005                        break;
5006                case 61:
5007                        block = "DMA1";
5008                        break;
5009                case 248:
5010                case 120:
5011                        block = "HDP";
5012                        break;
5013                default:
5014                        block = "unknown";
5015                        break;
5016                }
5017        }
5018
5019        printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5020               protections, vmid, addr,
5021               (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5022               block, mc_id);
5023}
5024
5025void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5026                 unsigned vm_id, uint64_t pd_addr)
5027{
5028        /* write new base address */
5029        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5030        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5031                                 WRITE_DATA_DST_SEL(0)));
5032
5033        if (vm_id < 8) {
5034                radeon_ring_write(ring,
5035                                  (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5036        } else {
5037                radeon_ring_write(ring,
5038                                  (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5039        }
5040        radeon_ring_write(ring, 0);
5041        radeon_ring_write(ring, pd_addr >> 12);
5042
5043        /* flush hdp cache */
5044        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5045        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5046                                 WRITE_DATA_DST_SEL(0)));
5047        radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5048        radeon_ring_write(ring, 0);
5049        radeon_ring_write(ring, 0x1);
5050
5051        /* bits 0-15 are the VM contexts0-15 */
5052        radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5053        radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5054                                 WRITE_DATA_DST_SEL(0)));
5055        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5056        radeon_ring_write(ring, 0);
5057        radeon_ring_write(ring, 1 << vm_id);
5058
5059        /* wait for the invalidate to complete */
5060        radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5061        radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
5062                                 WAIT_REG_MEM_ENGINE(0))); /* me */
5063        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5064        radeon_ring_write(ring, 0);
5065        radeon_ring_write(ring, 0); /* ref */
5066        radeon_ring_write(ring, 0); /* mask */
5067        radeon_ring_write(ring, 0x20); /* poll interval */
5068
5069        /* sync PFP to ME, otherwise we might get invalid PFP reads */
5070        radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5071        radeon_ring_write(ring, 0x0);
5072}
5073
5074/*
5075 *  Power and clock gating
5076 */
5077static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5078{
5079        int i;
5080
5081        for (i = 0; i < rdev->usec_timeout; i++) {
5082                if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5083                        break;
5084                udelay(1);
5085        }
5086
5087        for (i = 0; i < rdev->usec_timeout; i++) {
5088                if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5089                        break;
5090                udelay(1);
5091        }
5092}
5093
5094static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5095                                         bool enable)
5096{
5097        u32 tmp = RREG32(CP_INT_CNTL_RING0);
5098        u32 mask;
5099        int i;
5100
5101        if (enable)
5102                tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5103        else
5104                tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5105        WREG32(CP_INT_CNTL_RING0, tmp);
5106
5107        if (!enable) {
5108                /* read a gfx register */
5109                tmp = RREG32(DB_DEPTH_INFO);
5110
5111                mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5112                for (i = 0; i < rdev->usec_timeout; i++) {
5113                        if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5114                                break;
5115                        udelay(1);
5116                }
5117        }
5118}
5119
5120static void si_set_uvd_dcm(struct radeon_device *rdev,
5121                           bool sw_mode)
5122{
5123        u32 tmp, tmp2;
5124
5125        tmp = RREG32(UVD_CGC_CTRL);
5126        tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5127        tmp |= DCM | CG_DT(1) | CLK_OD(4);
5128
5129        if (sw_mode) {
5130                tmp &= ~0x7ffff800;
5131                tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5132        } else {
5133                tmp |= 0x7ffff800;
5134                tmp2 = 0;
5135        }
5136
5137        WREG32(UVD_CGC_CTRL, tmp);
5138        WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5139}
5140
5141void si_init_uvd_internal_cg(struct radeon_device *rdev)
5142{
5143        bool hw_mode = true;
5144
5145        if (hw_mode) {
5146                si_set_uvd_dcm(rdev, false);
5147        } else {
5148                u32 tmp = RREG32(UVD_CGC_CTRL);
5149                tmp &= ~DCM;
5150                WREG32(UVD_CGC_CTRL, tmp);
5151        }
5152}
5153
5154static u32 si_halt_rlc(struct radeon_device *rdev)
5155{
5156        u32 data, orig;
5157
5158        orig = data = RREG32(RLC_CNTL);
5159
5160        if (data & RLC_ENABLE) {
5161                data &= ~RLC_ENABLE;
5162                WREG32(RLC_CNTL, data);
5163
5164                si_wait_for_rlc_serdes(rdev);
5165        }
5166
5167        return orig;
5168}
5169
5170static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5171{
5172        u32 tmp;
5173
5174        tmp = RREG32(RLC_CNTL);
5175        if (tmp != rlc)
5176                WREG32(RLC_CNTL, rlc);
5177}
5178
5179static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5180{
5181        u32 data, orig;
5182
5183        orig = data = RREG32(DMA_PG);
5184        if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5185                data |= PG_CNTL_ENABLE;
5186        else
5187                data &= ~PG_CNTL_ENABLE;
5188        if (orig != data)
5189                WREG32(DMA_PG, data);
5190}
5191
5192static void si_init_dma_pg(struct radeon_device *rdev)
5193{
5194        u32 tmp;
5195
5196        WREG32(DMA_PGFSM_WRITE,  0x00002000);
5197        WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5198
5199        for (tmp = 0; tmp < 5; tmp++)
5200                WREG32(DMA_PGFSM_WRITE, 0);
5201}
5202
5203static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5204                               bool enable)
5205{
5206        u32 tmp;
5207
5208        if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5209                tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5210                WREG32(RLC_TTOP_D, tmp);
5211
5212                tmp = RREG32(RLC_PG_CNTL);
5213                tmp |= GFX_PG_ENABLE;
5214                WREG32(RLC_PG_CNTL, tmp);
5215
5216                tmp = RREG32(RLC_AUTO_PG_CTRL);
5217                tmp |= AUTO_PG_EN;
5218                WREG32(RLC_AUTO_PG_CTRL, tmp);
5219        } else {
5220                tmp = RREG32(RLC_AUTO_PG_CTRL);
5221                tmp &= ~AUTO_PG_EN;
5222                WREG32(RLC_AUTO_PG_CTRL, tmp);
5223
5224                tmp = RREG32(DB_RENDER_CONTROL);
5225        }
5226}
5227
5228static void si_init_gfx_cgpg(struct radeon_device *rdev)
5229{
5230        u32 tmp;
5231
5232        WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5233
5234        tmp = RREG32(RLC_PG_CNTL);
5235        tmp |= GFX_PG_SRC;
5236        WREG32(RLC_PG_CNTL, tmp);
5237
5238        WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5239
5240        tmp = RREG32(RLC_AUTO_PG_CTRL);
5241
5242        tmp &= ~GRBM_REG_SGIT_MASK;
5243        tmp |= GRBM_REG_SGIT(0x700);
5244        tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5245        WREG32(RLC_AUTO_PG_CTRL, tmp);
5246}
5247
5248static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5249{
5250        u32 mask = 0, tmp, tmp1;
5251        int i;
5252
5253        si_select_se_sh(rdev, se, sh);
5254        tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5255        tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5256        si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5257
5258        tmp &= 0xffff0000;
5259
5260        tmp |= tmp1;
5261        tmp >>= 16;
5262
5263        for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5264                mask <<= 1;
5265                mask |= 1;
5266        }
5267
5268        return (~tmp) & mask;
5269}
5270
5271static void si_init_ao_cu_mask(struct radeon_device *rdev)
5272{
5273        u32 i, j, k, active_cu_number = 0;
5274        u32 mask, counter, cu_bitmap;
5275        u32 tmp = 0;
5276
5277        for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5278                for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5279                        mask = 1;
5280                        cu_bitmap = 0;
5281                        counter  = 0;
5282                        for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5283                                if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5284                                        if (counter < 2)
5285                                                cu_bitmap |= mask;
5286                                        counter++;
5287                                }
5288                                mask <<= 1;
5289                        }
5290
5291                        active_cu_number += counter;
5292                        tmp |= (cu_bitmap << (i * 16 + j * 8));
5293                }
5294        }
5295
5296        WREG32(RLC_PG_AO_CU_MASK, tmp);
5297
5298        tmp = RREG32(RLC_MAX_PG_CU);
5299        tmp &= ~MAX_PU_CU_MASK;
5300        tmp |= MAX_PU_CU(active_cu_number);
5301        WREG32(RLC_MAX_PG_CU, tmp);
5302}
5303
5304static void si_enable_cgcg(struct radeon_device *rdev,
5305                           bool enable)
5306{
5307        u32 data, orig, tmp;
5308
5309        orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5310
5311        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5312                si_enable_gui_idle_interrupt(rdev, true);
5313
5314                WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5315
5316                tmp = si_halt_rlc(rdev);
5317
5318                WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5319                WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5320                WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5321
5322                si_wait_for_rlc_serdes(rdev);
5323
5324                si_update_rlc(rdev, tmp);
5325
5326                WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5327
5328                data |= CGCG_EN | CGLS_EN;
5329        } else {
5330                si_enable_gui_idle_interrupt(rdev, false);
5331
5332                RREG32(CB_CGTT_SCLK_CTRL);
5333                RREG32(CB_CGTT_SCLK_CTRL);
5334                RREG32(CB_CGTT_SCLK_CTRL);
5335                RREG32(CB_CGTT_SCLK_CTRL);
5336
5337                data &= ~(CGCG_EN | CGLS_EN);
5338        }
5339
5340        if (orig != data)
5341                WREG32(RLC_CGCG_CGLS_CTRL, data);
5342}
5343
5344static void si_enable_mgcg(struct radeon_device *rdev,
5345                           bool enable)
5346{
5347        u32 data, orig, tmp = 0;
5348
5349        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5350                orig = data = RREG32(CGTS_SM_CTRL_REG);
5351                data = 0x96940200;
5352                if (orig != data)
5353                        WREG32(CGTS_SM_CTRL_REG, data);
5354
5355                if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5356                        orig = data = RREG32(CP_MEM_SLP_CNTL);
5357                        data |= CP_MEM_LS_EN;
5358                        if (orig != data)
5359                                WREG32(CP_MEM_SLP_CNTL, data);
5360                }
5361
5362                orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5363                data &= 0xffffffc0;
5364                if (orig != data)
5365                        WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5366
5367                tmp = si_halt_rlc(rdev);
5368
5369                WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5370                WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5371                WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5372
5373                si_update_rlc(rdev, tmp);
5374        } else {
5375                orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5376                data |= 0x00000003;
5377                if (orig != data)
5378                        WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5379
5380                data = RREG32(CP_MEM_SLP_CNTL);
5381                if (data & CP_MEM_LS_EN) {
5382                        data &= ~CP_MEM_LS_EN;
5383                        WREG32(CP_MEM_SLP_CNTL, data);
5384                }
5385                orig = data = RREG32(CGTS_SM_CTRL_REG);
5386                data |= LS_OVERRIDE | OVERRIDE;
5387                if (orig != data)
5388                        WREG32(CGTS_SM_CTRL_REG, data);
5389
5390                tmp = si_halt_rlc(rdev);
5391
5392                WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5393                WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5394                WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5395
5396                si_update_rlc(rdev, tmp);
5397        }
5398}
5399
5400static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5401                               bool enable)
5402{
5403        u32 orig, data, tmp;
5404
5405        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5406                tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5407                tmp |= 0x3fff;
5408                WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5409
5410                orig = data = RREG32(UVD_CGC_CTRL);
5411                data |= DCM;
5412                if (orig != data)
5413                        WREG32(UVD_CGC_CTRL, data);
5414
5415                WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5416                WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5417        } else {
5418                tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5419                tmp &= ~0x3fff;
5420                WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5421
5422                orig = data = RREG32(UVD_CGC_CTRL);
5423                data &= ~DCM;
5424                if (orig != data)
5425                        WREG32(UVD_CGC_CTRL, data);
5426
5427                WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5428                WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5429        }
5430}
5431
5432static const u32 mc_cg_registers[] =
5433{
5434        MC_HUB_MISC_HUB_CG,
5435        MC_HUB_MISC_SIP_CG,
5436        MC_HUB_MISC_VM_CG,
5437        MC_XPB_CLK_GAT,
5438        ATC_MISC_CG,
5439        MC_CITF_MISC_WR_CG,
5440        MC_CITF_MISC_RD_CG,
5441        MC_CITF_MISC_VM_CG,
5442        VM_L2_CG,
5443};
5444
5445static void si_enable_mc_ls(struct radeon_device *rdev,
5446                            bool enable)
5447{
5448        int i;
5449        u32 orig, data;
5450
5451        for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5452                orig = data = RREG32(mc_cg_registers[i]);
5453                if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5454                        data |= MC_LS_ENABLE;
5455                else
5456                        data &= ~MC_LS_ENABLE;
5457                if (data != orig)
5458                        WREG32(mc_cg_registers[i], data);
5459        }
5460}
5461
5462static void si_enable_mc_mgcg(struct radeon_device *rdev,
5463                               bool enable)
5464{
5465        int i;
5466        u32 orig, data;
5467
5468        for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5469                orig = data = RREG32(mc_cg_registers[i]);
5470                if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5471                        data |= MC_CG_ENABLE;
5472                else
5473                        data &= ~MC_CG_ENABLE;
5474                if (data != orig)
5475                        WREG32(mc_cg_registers[i], data);
5476        }
5477}
5478
5479static void si_enable_dma_mgcg(struct radeon_device *rdev,
5480                               bool enable)
5481{
5482        u32 orig, data, offset;
5483        int i;
5484
5485        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5486                for (i = 0; i < 2; i++) {
5487                        if (i == 0)
5488                                offset = DMA0_REGISTER_OFFSET;
5489                        else
5490                                offset = DMA1_REGISTER_OFFSET;
5491                        orig = data = RREG32(DMA_POWER_CNTL + offset);
5492                        data &= ~MEM_POWER_OVERRIDE;
5493                        if (data != orig)
5494                                WREG32(DMA_POWER_CNTL + offset, data);
5495                        WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5496                }
5497        } else {
5498                for (i = 0; i < 2; i++) {
5499                        if (i == 0)
5500                                offset = DMA0_REGISTER_OFFSET;
5501                        else
5502                                offset = DMA1_REGISTER_OFFSET;
5503                        orig = data = RREG32(DMA_POWER_CNTL + offset);
5504                        data |= MEM_POWER_OVERRIDE;
5505                        if (data != orig)
5506                                WREG32(DMA_POWER_CNTL + offset, data);
5507
5508                        orig = data = RREG32(DMA_CLK_CTRL + offset);
5509                        data = 0xff000000;
5510                        if (data != orig)
5511                                WREG32(DMA_CLK_CTRL + offset, data);
5512                }
5513        }
5514}
5515
5516static void si_enable_bif_mgls(struct radeon_device *rdev,
5517                               bool enable)
5518{
5519        u32 orig, data;
5520
5521        orig = data = RREG32_PCIE(PCIE_CNTL2);
5522
5523        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5524                data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5525                        REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5526        else
5527                data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5528                          REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5529
5530        if (orig != data)
5531                WREG32_PCIE(PCIE_CNTL2, data);
5532}
5533
5534static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5535                               bool enable)
5536{
5537        u32 orig, data;
5538
5539        orig = data = RREG32(HDP_HOST_PATH_CNTL);
5540
5541        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5542                data &= ~CLOCK_GATING_DIS;
5543        else
5544                data |= CLOCK_GATING_DIS;
5545
5546        if (orig != data)
5547                WREG32(HDP_HOST_PATH_CNTL, data);
5548}
5549
5550static void si_enable_hdp_ls(struct radeon_device *rdev,
5551                             bool enable)
5552{
5553        u32 orig, data;
5554
5555        orig = data = RREG32(HDP_MEM_POWER_LS);
5556
5557        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5558                data |= HDP_LS_ENABLE;
5559        else
5560                data &= ~HDP_LS_ENABLE;
5561
5562        if (orig != data)
5563                WREG32(HDP_MEM_POWER_LS, data);
5564}
5565
5566static void si_update_cg(struct radeon_device *rdev,
5567                         u32 block, bool enable)
5568{
5569        if (block & RADEON_CG_BLOCK_GFX) {
5570                si_enable_gui_idle_interrupt(rdev, false);
5571                /* order matters! */
5572                if (enable) {
5573                        si_enable_mgcg(rdev, true);
5574                        si_enable_cgcg(rdev, true);
5575                } else {
5576                        si_enable_cgcg(rdev, false);
5577                        si_enable_mgcg(rdev, false);
5578                }
5579                si_enable_gui_idle_interrupt(rdev, true);
5580        }
5581
5582        if (block & RADEON_CG_BLOCK_MC) {
5583                si_enable_mc_mgcg(rdev, enable);
5584                si_enable_mc_ls(rdev, enable);
5585        }
5586
5587        if (block & RADEON_CG_BLOCK_SDMA) {
5588                si_enable_dma_mgcg(rdev, enable);
5589        }
5590
5591        if (block & RADEON_CG_BLOCK_BIF) {
5592                si_enable_bif_mgls(rdev, enable);
5593        }
5594
5595        if (block & RADEON_CG_BLOCK_UVD) {
5596                if (rdev->has_uvd) {
5597                        si_enable_uvd_mgcg(rdev, enable);
5598                }
5599        }
5600
5601        if (block & RADEON_CG_BLOCK_HDP) {
5602                si_enable_hdp_mgcg(rdev, enable);
5603                si_enable_hdp_ls(rdev, enable);
5604        }
5605}
5606
5607static void si_init_cg(struct radeon_device *rdev)
5608{
5609        si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5610                            RADEON_CG_BLOCK_MC |
5611                            RADEON_CG_BLOCK_SDMA |
5612                            RADEON_CG_BLOCK_BIF |
5613                            RADEON_CG_BLOCK_HDP), true);
5614        if (rdev->has_uvd) {
5615                si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5616                si_init_uvd_internal_cg(rdev);
5617        }
5618}
5619
5620static void si_fini_cg(struct radeon_device *rdev)
5621{
5622        if (rdev->has_uvd) {
5623                si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5624        }
5625        si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5626                            RADEON_CG_BLOCK_MC |
5627                            RADEON_CG_BLOCK_SDMA |
5628                            RADEON_CG_BLOCK_BIF |
5629                            RADEON_CG_BLOCK_HDP), false);
5630}
5631
5632u32 si_get_csb_size(struct radeon_device *rdev)
5633{
5634        u32 count = 0;
5635        const struct cs_section_def *sect = NULL;
5636        const struct cs_extent_def *ext = NULL;
5637
5638        if (rdev->rlc.cs_data == NULL)
5639                return 0;
5640
5641        /* begin clear state */
5642        count += 2;
5643        /* context control state */
5644        count += 3;
5645
5646        for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5647                for (ext = sect->section; ext->extent != NULL; ++ext) {
5648                        if (sect->id == SECT_CONTEXT)
5649                                count += 2 + ext->reg_count;
5650                        else
5651                                return 0;
5652                }
5653        }
5654        /* pa_sc_raster_config */
5655        count += 3;
5656        /* end clear state */
5657        count += 2;
5658        /* clear state */
5659        count += 2;
5660
5661        return count;
5662}
5663
5664void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5665{
5666        u32 count = 0, i;
5667        const struct cs_section_def *sect = NULL;
5668        const struct cs_extent_def *ext = NULL;
5669
5670        if (rdev->rlc.cs_data == NULL)
5671                return;
5672        if (buffer == NULL)
5673                return;
5674
5675        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5676        buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5677
5678        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5679        buffer[count++] = cpu_to_le32(0x80000000);
5680        buffer[count++] = cpu_to_le32(0x80000000);
5681
5682        for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5683                for (ext = sect->section; ext->extent != NULL; ++ext) {
5684                        if (sect->id == SECT_CONTEXT) {
5685                                buffer[count++] =
5686                                        cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5687                                buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5688                                for (i = 0; i < ext->reg_count; i++)
5689                                        buffer[count++] = cpu_to_le32(ext->extent[i]);
5690                        } else {
5691                                return;
5692                        }
5693                }
5694        }
5695
5696        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5697        buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5698        switch (rdev->family) {
5699        case CHIP_TAHITI:
5700        case CHIP_PITCAIRN:
5701                buffer[count++] = cpu_to_le32(0x2a00126a);
5702                break;
5703        case CHIP_VERDE:
5704                buffer[count++] = cpu_to_le32(0x0000124a);
5705                break;
5706        case CHIP_OLAND:
5707                buffer[count++] = cpu_to_le32(0x00000082);
5708                break;
5709        case CHIP_HAINAN:
5710                buffer[count++] = cpu_to_le32(0x00000000);
5711                break;
5712        default:
5713                buffer[count++] = cpu_to_le32(0x00000000);
5714                break;
5715        }
5716
5717        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5718        buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5719
5720        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5721        buffer[count++] = cpu_to_le32(0);
5722}
5723
5724static void si_init_pg(struct radeon_device *rdev)
5725{
5726        if (rdev->pg_flags) {
5727                if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5728                        si_init_dma_pg(rdev);
5729                }
5730                si_init_ao_cu_mask(rdev);
5731                if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5732                        si_init_gfx_cgpg(rdev);
5733                } else {
5734                        WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5735                        WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5736                }
5737                si_enable_dma_pg(rdev, true);
5738                si_enable_gfx_cgpg(rdev, true);
5739        } else {
5740                WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5741                WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5742        }
5743}
5744
5745static void si_fini_pg(struct radeon_device *rdev)
5746{
5747        if (rdev->pg_flags) {
5748                si_enable_dma_pg(rdev, false);
5749                si_enable_gfx_cgpg(rdev, false);
5750        }
5751}
5752
5753/*
5754 * RLC
5755 */
5756void si_rlc_reset(struct radeon_device *rdev)
5757{
5758        u32 tmp = RREG32(GRBM_SOFT_RESET);
5759
5760        tmp |= SOFT_RESET_RLC;
5761        WREG32(GRBM_SOFT_RESET, tmp);
5762        udelay(50);
5763        tmp &= ~SOFT_RESET_RLC;
5764        WREG32(GRBM_SOFT_RESET, tmp);
5765        udelay(50);
5766}
5767
5768static void si_rlc_stop(struct radeon_device *rdev)
5769{
5770        WREG32(RLC_CNTL, 0);
5771
5772        si_enable_gui_idle_interrupt(rdev, false);
5773
5774        si_wait_for_rlc_serdes(rdev);
5775}
5776
5777static void si_rlc_start(struct radeon_device *rdev)
5778{
5779        WREG32(RLC_CNTL, RLC_ENABLE);
5780
5781        si_enable_gui_idle_interrupt(rdev, true);
5782
5783        udelay(50);
5784}
5785
5786static bool si_lbpw_supported(struct radeon_device *rdev)
5787{
5788        u32 tmp;
5789
5790        /* Enable LBPW only for DDR3 */
5791        tmp = RREG32(MC_SEQ_MISC0);
5792        if ((tmp & 0xF0000000) == 0xB0000000)
5793                return true;
5794        return false;
5795}
5796
5797static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5798{
5799        u32 tmp;
5800
5801        tmp = RREG32(RLC_LB_CNTL);
5802        if (enable)
5803                tmp |= LOAD_BALANCE_ENABLE;
5804        else
5805                tmp &= ~LOAD_BALANCE_ENABLE;
5806        WREG32(RLC_LB_CNTL, tmp);
5807
5808        if (!enable) {
5809                si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5810                WREG32(SPI_LB_CU_MASK, 0x00ff);
5811        }
5812}
5813
5814static int si_rlc_resume(struct radeon_device *rdev)
5815{
5816        u32 i;
5817
5818        if (!rdev->rlc_fw)
5819                return -EINVAL;
5820
5821        si_rlc_stop(rdev);
5822
5823        si_rlc_reset(rdev);
5824
5825        si_init_pg(rdev);
5826
5827        si_init_cg(rdev);
5828
5829        WREG32(RLC_RL_BASE, 0);
5830        WREG32(RLC_RL_SIZE, 0);
5831        WREG32(RLC_LB_CNTL, 0);
5832        WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5833        WREG32(RLC_LB_CNTR_INIT, 0);
5834        WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5835
5836        WREG32(RLC_MC_CNTL, 0);
5837        WREG32(RLC_UCODE_CNTL, 0);
5838
5839        if (rdev->new_fw) {
5840                const struct rlc_firmware_header_v1_0 *hdr =
5841                        (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5842                u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5843                const __le32 *fw_data = (const __le32 *)
5844                        (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5845
5846                radeon_ucode_print_rlc_hdr(&hdr->header);
5847
5848                for (i = 0; i < fw_size; i++) {
5849                        WREG32(RLC_UCODE_ADDR, i);
5850                        WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5851                }
5852        } else {
5853                const __be32 *fw_data =
5854                        (const __be32 *)rdev->rlc_fw->data;
5855                for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5856                        WREG32(RLC_UCODE_ADDR, i);
5857                        WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5858                }
5859        }
5860        WREG32(RLC_UCODE_ADDR, 0);
5861
5862        si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5863
5864        si_rlc_start(rdev);
5865
5866        return 0;
5867}
5868
5869static void si_enable_interrupts(struct radeon_device *rdev)
5870{
5871        u32 ih_cntl = RREG32(IH_CNTL);
5872        u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5873
5874        ih_cntl |= ENABLE_INTR;
5875        ih_rb_cntl |= IH_RB_ENABLE;
5876        WREG32(IH_CNTL, ih_cntl);
5877        WREG32(IH_RB_CNTL, ih_rb_cntl);
5878        rdev->ih.enabled = true;
5879}
5880
5881static void si_disable_interrupts(struct radeon_device *rdev)
5882{
5883        u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5884        u32 ih_cntl = RREG32(IH_CNTL);
5885
5886        ih_rb_cntl &= ~IH_RB_ENABLE;
5887        ih_cntl &= ~ENABLE_INTR;
5888        WREG32(IH_RB_CNTL, ih_rb_cntl);
5889        WREG32(IH_CNTL, ih_cntl);
5890        /* set rptr, wptr to 0 */
5891        WREG32(IH_RB_RPTR, 0);
5892        WREG32(IH_RB_WPTR, 0);
5893        rdev->ih.enabled = false;
5894        rdev->ih.rptr = 0;
5895}
5896
5897static void si_disable_interrupt_state(struct radeon_device *rdev)
5898{
5899        u32 tmp;
5900
5901        tmp = RREG32(CP_INT_CNTL_RING0) &
5902                (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5903        WREG32(CP_INT_CNTL_RING0, tmp);
5904        WREG32(CP_INT_CNTL_RING1, 0);
5905        WREG32(CP_INT_CNTL_RING2, 0);
5906        tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5907        WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5908        tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5909        WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5910        WREG32(GRBM_INT_CNTL, 0);
5911        WREG32(SRBM_INT_CNTL, 0);
5912        if (rdev->num_crtc >= 2) {
5913                WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5914                WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5915        }
5916        if (rdev->num_crtc >= 4) {
5917                WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5918                WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5919        }
5920        if (rdev->num_crtc >= 6) {
5921                WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5922                WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5923        }
5924
5925        if (rdev->num_crtc >= 2) {
5926                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5927                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5928        }
5929        if (rdev->num_crtc >= 4) {
5930                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5931                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5932        }
5933        if (rdev->num_crtc >= 6) {
5934                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5935                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5936        }
5937
5938        if (!ASIC_IS_NODCE(rdev)) {
5939                WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5940
5941                tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5942                WREG32(DC_HPD1_INT_CONTROL, tmp);
5943                tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5944                WREG32(DC_HPD2_INT_CONTROL, tmp);
5945                tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5946                WREG32(DC_HPD3_INT_CONTROL, tmp);
5947                tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5948                WREG32(DC_HPD4_INT_CONTROL, tmp);
5949                tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5950                WREG32(DC_HPD5_INT_CONTROL, tmp);
5951                tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5952                WREG32(DC_HPD6_INT_CONTROL, tmp);
5953        }
5954}
5955
5956static int si_irq_init(struct radeon_device *rdev)
5957{
5958        int ret = 0;
5959        int rb_bufsz;
5960        u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5961
5962        /* allocate ring */
5963        ret = r600_ih_ring_alloc(rdev);
5964        if (ret)
5965                return ret;
5966
5967        /* disable irqs */
5968        si_disable_interrupts(rdev);
5969
5970        /* init rlc */
5971        ret = si_rlc_resume(rdev);
5972        if (ret) {
5973                r600_ih_ring_fini(rdev);
5974                return ret;
5975        }
5976
5977        /* setup interrupt control */
5978        /* set dummy read address to ring address */
5979        WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5980        interrupt_cntl = RREG32(INTERRUPT_CNTL);
5981        /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5982         * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5983         */
5984        interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5985        /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5986        interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5987        WREG32(INTERRUPT_CNTL, interrupt_cntl);
5988
5989        WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5990        rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5991
5992        ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5993                      IH_WPTR_OVERFLOW_CLEAR |
5994                      (rb_bufsz << 1));
5995
5996        if (rdev->wb.enabled)
5997                ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5998
5999        /* set the writeback address whether it's enabled or not */
6000        WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);

6001        WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6002
6003        WREG32(IH_RB_CNTL, ih_rb_cntl);
6004
6005        /* set rptr, wptr to 0 */
6006        WREG32(IH_RB_RPTR, 0);
6007        WREG32(IH_RB_WPTR, 0);
6008
6009        /* Default settings for IH_CNTL (disabled at first) */
6010        ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6011        /* RPTR_REARM only works if msi's are enabled */
6012        if (rdev->msi_enabled)
6013                ih_cntl |= RPTR_REARM;
6014        WREG32(IH_CNTL, ih_cntl);
6015
6016        /* force the active interrupt state to all disabled */
6017        si_disable_interrupt_state(rdev);
6018
6019        pci_set_master(rdev->pdev);
6020
6021        /* enable irqs */
6022        si_enable_interrupts(rdev);
6023
6024        return ret;
6025}
6026
6027int si_irq_set(struct radeon_device *rdev)
6028{
6029        u32 cp_int_cntl;
6030        u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
6031        u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6032        u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
6033        u32 grbm_int_cntl = 0;
6034        u32 dma_cntl, dma_cntl1;
6035        u32 thermal_int = 0;
6036
6037        if (!rdev->irq.installed) {
6038                WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6039                return -EINVAL;
6040        }
6041        /* don't enable anything if the ih is disabled */
6042        if (!rdev->ih.enabled) {
6043                si_disable_interrupts(rdev);
6044                /* force the active interrupt state to all disabled */
6045                si_disable_interrupt_state(rdev);
6046                return 0;
6047        }
6048
6049        cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6050                (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6051
6052        if (!ASIC_IS_NODCE(rdev)) {
6053                hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6054                hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6055                hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6056                hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6057                hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6058                hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6059        }
6060
6061        dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6062        dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6063
6064        thermal_int = RREG32(CG_THERMAL_INT) &
6065                ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6066
6067        /* enable CP interrupts on all rings */
6068        if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6069                DRM_DEBUG("si_irq_set: sw int gfx\n");
6070                cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6071        }
6072        if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6073                DRM_DEBUG("si_irq_set: sw int cp1\n");
6074                cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6075        }
6076        if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6077                DRM_DEBUG("si_irq_set: sw int cp2\n");
6078                cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6079        }
6080        if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6081                DRM_DEBUG("si_irq_set: sw int dma\n");
6082                dma_cntl |= TRAP_ENABLE;
6083        }
6084
6085        if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6086                DRM_DEBUG("si_irq_set: sw int dma1\n");
6087                dma_cntl1 |= TRAP_ENABLE;
6088        }
6089        if (rdev->irq.crtc_vblank_int[0] ||
6090            atomic_read(&rdev->irq.pflip[0])) {
6091                DRM_DEBUG("si_irq_set: vblank 0\n");
6092                crtc1 |= VBLANK_INT_MASK;
6093        }
6094        if (rdev->irq.crtc_vblank_int[1] ||
6095            atomic_read(&rdev->irq.pflip[1])) {
6096                DRM_DEBUG("si_irq_set: vblank 1\n");
6097                crtc2 |= VBLANK_INT_MASK;
6098        }
6099        if (rdev->irq.crtc_vblank_int[2] ||
6100            atomic_read(&rdev->irq.pflip[2])) {
6101                DRM_DEBUG("si_irq_set: vblank 2\n");
6102                crtc3 |= VBLANK_INT_MASK;
6103        }
6104        if (rdev->irq.crtc_vblank_int[3] ||
6105            atomic_read(&rdev->irq.pflip[3])) {
6106                DRM_DEBUG("si_irq_set: vblank 3\n");
6107                crtc4 |= VBLANK_INT_MASK;
6108        }
6109        if (rdev->irq.crtc_vblank_int[4] ||
6110            atomic_read(&rdev->irq.pflip[4])) {
6111                DRM_DEBUG("si_irq_set: vblank 4\n");
6112                crtc5 |= VBLANK_INT_MASK;
6113        }
6114        if (rdev->irq.crtc_vblank_int[5] ||
6115            atomic_read(&rdev->irq.pflip[5])) {
6116                DRM_DEBUG("si_irq_set: vblank 5\n");
6117                crtc6 |= VBLANK_INT_MASK;
6118        }
6119        if (rdev->irq.hpd[0]) {
6120                DRM_DEBUG("si_irq_set: hpd 1\n");
6121                hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6122        }
6123        if (rdev->irq.hpd[1]) {
6124                DRM_DEBUG("si_irq_set: hpd 2\n");
6125                hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6126        }
6127        if (rdev->irq.hpd[2]) {
6128                DRM_DEBUG("si_irq_set: hpd 3\n");
6129                hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6130        }
6131        if (rdev->irq.hpd[3]) {
6132                DRM_DEBUG("si_irq_set: hpd 4\n");
6133                hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6134        }
6135        if (rdev->irq.hpd[4]) {
6136                DRM_DEBUG("si_irq_set: hpd 5\n");
6137                hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6138        }
6139        if (rdev->irq.hpd[5]) {
6140                DRM_DEBUG("si_irq_set: hpd 6\n");
6141                hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6142        }
6143
6144        WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6145        WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6146        WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6147
6148        WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6149        WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6150
6151        WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6152
6153        if (rdev->irq.dpm_thermal) {
6154                DRM_DEBUG("dpm thermal\n");
6155                thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6156        }
6157
6158        if (rdev->num_crtc >= 2) {
6159                WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6160                WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6161        }
6162        if (rdev->num_crtc >= 4) {
6163                WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6164                WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6165        }
6166        if (rdev->num_crtc >= 6) {
6167                WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6168                WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6169        }
6170
6171        if (rdev->num_crtc >= 2) {
6172                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
6173                       GRPH_PFLIP_INT_MASK);
6174                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
6175                       GRPH_PFLIP_INT_MASK);
6176        }
6177        if (rdev->num_crtc >= 4) {
6178                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
6179                       GRPH_PFLIP_INT_MASK);
6180                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
6181                       GRPH_PFLIP_INT_MASK);
6182        }
6183        if (rdev->num_crtc >= 6) {
6184                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
6185                       GRPH_PFLIP_INT_MASK);
6186                WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
6187                       GRPH_PFLIP_INT_MASK);
6188        }
6189
6190        if (!ASIC_IS_NODCE(rdev)) {
6191                WREG32(DC_HPD1_INT_CONTROL, hpd1);
6192                WREG32(DC_HPD2_INT_CONTROL, hpd2);
6193                WREG32(DC_HPD3_INT_CONTROL, hpd3);
6194                WREG32(DC_HPD4_INT_CONTROL, hpd4);
6195                WREG32(DC_HPD5_INT_CONTROL, hpd5);
6196                WREG32(DC_HPD6_INT_CONTROL, hpd6);
6197        }
6198
6199        WREG32(CG_THERMAL_INT, thermal_int);
6200
6201        /* posting read */
6202        RREG32(SRBM_STATUS);
6203
6204        return 0;
6205}
6206
6207static inline void si_irq_ack(struct radeon_device *rdev)
6208{
6209        u32 tmp;
6210
6211        if (ASIC_IS_NODCE(rdev))
6212                return;
6213
6214        rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6215        rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6216        rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6217        rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6218        rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6219        rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6220        rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
6221        rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
6222        if (rdev->num_crtc >= 4) {
6223                rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
6224                rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
6225        }
6226        if (rdev->num_crtc >= 6) {
6227                rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
6228                rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
6229        }
6230
6231        if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
6232                WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6233        if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
6234                WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6235        if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
6236                WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6237        if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
6238                WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6239        if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6240                WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6241        if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6242                WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6243
6244        if (rdev->num_crtc >= 4) {
6245                if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
6246                        WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6247                if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
6248                        WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6249                if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6250                        WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6251                if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6252                        WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6253                if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6254                        WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6255                if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6256                        WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6257        }
6258
6259        if (rdev->num_crtc >= 6) {
6260                if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6261                        WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6262                if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6263                        WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6264                if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6265                        WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6266                if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6267                        WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6268                if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6269                        WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6270                if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6271                        WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6272        }
6273
6274        if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6275                tmp = RREG32(DC_HPD1_INT_CONTROL);
6276                tmp |= DC_HPDx_INT_ACK;
6277                WREG32(DC_HPD1_INT_CONTROL, tmp);
6278        }
6279        if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6280                tmp = RREG32(DC_HPD2_INT_CONTROL);
6281                tmp |= DC_HPDx_INT_ACK;
6282                WREG32(DC_HPD2_INT_CONTROL, tmp);
6283        }
6284        if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6285                tmp = RREG32(DC_HPD3_INT_CONTROL);
6286                tmp |= DC_HPDx_INT_ACK;
6287                WREG32(DC_HPD3_INT_CONTROL, tmp);
6288        }
6289        if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6290                tmp = RREG32(DC_HPD4_INT_CONTROL);
6291                tmp |= DC_HPDx_INT_ACK;
6292                WREG32(DC_HPD4_INT_CONTROL, tmp);
6293        }
6294        if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6295                tmp = RREG32(DC_HPD5_INT_CONTROL);
6296                tmp |= DC_HPDx_INT_ACK;
6297                WREG32(DC_HPD5_INT_CONTROL, tmp);
6298        }
6299        if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6300                tmp = RREG32(DC_HPD5_INT_CONTROL);
6301                tmp |= DC_HPDx_INT_ACK;
6302                WREG32(DC_HPD6_INT_CONTROL, tmp);
6303        }
6304
6305        if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
6306                tmp = RREG32(DC_HPD1_INT_CONTROL);
6307                tmp |= DC_HPDx_RX_INT_ACK;
6308                WREG32(DC_HPD1_INT_CONTROL, tmp);
6309        }
6310        if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
6311                tmp = RREG32(DC_HPD2_INT_CONTROL);
6312                tmp |= DC_HPDx_RX_INT_ACK;
6313                WREG32(DC_HPD2_INT_CONTROL, tmp);
6314        }
6315        if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
6316                tmp = RREG32(DC_HPD3_INT_CONTROL);
6317                tmp |= DC_HPDx_RX_INT_ACK;
6318                WREG32(DC_HPD3_INT_CONTROL, tmp);
6319        }
6320        if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
6321                tmp = RREG32(DC_HPD4_INT_CONTROL);
6322                tmp |= DC_HPDx_RX_INT_ACK;
6323                WREG32(DC_HPD4_INT_CONTROL, tmp);
6324        }
6325        if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
6326                tmp = RREG32(DC_HPD5_INT_CONTROL);
6327                tmp |= DC_HPDx_RX_INT_ACK;
6328                WREG32(DC_HPD5_INT_CONTROL, tmp);
6329        }
6330        if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
6331                tmp = RREG32(DC_HPD5_INT_CONTROL);
6332                tmp |= DC_HPDx_RX_INT_ACK;
6333                WREG32(DC_HPD6_INT_CONTROL, tmp);
6334        }
6335}
6336
6337static void si_irq_disable(struct radeon_device *rdev)
6338{
6339        si_disable_interrupts(rdev);
6340        /* Wait and acknowledge irq */
6341        mdelay(1);
6342        si_irq_ack(rdev);
6343        si_disable_interrupt_state(rdev);
6344}
6345
6346static void si_irq_suspend(struct radeon_device *rdev)
6347{
6348        si_irq_disable(rdev);
6349        si_rlc_stop(rdev);
6350}
6351
6352static void si_irq_fini(struct radeon_device *rdev)
6353{
6354        si_irq_suspend(rdev);
6355        r600_ih_ring_fini(rdev);
6356}
6357
6358static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6359{
6360        u32 wptr, tmp;
6361
6362        if (rdev->wb.enabled)
6363                wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6364        else
6365                wptr = RREG32(IH_RB_WPTR);
6366
6367        if (wptr & RB_OVERFLOW) {
6368                wptr &= ~RB_OVERFLOW;
6369                /* When a ring buffer overflow happen start parsing interrupt
6370                 * from the last not overwritten vector (wptr + 16). Hopefully
6371                 * this should allow us to catchup.
6372                 */
6373                dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6374                         wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6375                rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6376                tmp = RREG32(IH_RB_CNTL);
6377                tmp |= IH_WPTR_OVERFLOW_CLEAR;
6378                WREG32(IH_RB_CNTL, tmp);
6379        }
6380        return (wptr & rdev->ih.ptr_mask);
6381}
6382
6383/*        SI IV Ring
6384 * Each IV ring entry is 128 bits:
6385 * [7:0]    - interrupt source id
6386 * [31:8]   - reserved
6387 * [59:32]  - interrupt source data
6388 * [63:60]  - reserved
6389 * [71:64]  - RINGID
6390 * [79:72]  - VMID
6391 * [127:80] - reserved
6392 */
6393int si_irq_process(struct radeon_device *rdev)
6394{
6395        u32 wptr;
6396        u32 rptr;
6397        u32 src_id, src_data, ring_id;
6398        u32 ring_index;
6399        bool queue_hotplug = false;
6400        bool queue_dp = false;
6401        bool queue_thermal = false;
6402        u32 status, addr;
6403
6404        if (!rdev->ih.enabled || rdev->shutdown)
6405                return IRQ_NONE;
6406
6407        wptr = si_get_ih_wptr(rdev);
6408
6409restart_ih:
6410        /* is somebody else already processing irqs? */
6411        if (atomic_xchg(&rdev->ih.lock, 1))
6412                return IRQ_NONE;
6413
6414        rptr = rdev->ih.rptr;
6415        DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6416
6417        /* Order reading of wptr vs. reading of IH ring data */
6418        rmb();
6419
6420        /* display interrupts */
6421        si_irq_ack(rdev);
6422
6423        while (rptr != wptr) {
6424                /* wptr/rptr are in bytes! */
6425                ring_index = rptr / 4;
6426                src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6427                src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6428                ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6429
6430                switch (src_id) {
6431                case 1: /* D1 vblank/vline */
6432                        switch (src_data) {
6433                        case 0: /* D1 vblank */
6434                                if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
6435                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6436
6437                                if (rdev->irq.crtc_vblank_int[0]) {
6438                                        drm_handle_vblank(rdev->ddev, 0);
6439                                        rdev->pm.vblank_sync = true;
6440                                        wake_up(&rdev->irq.vblank_queue);
6441                                }
6442                                if (atomic_read(&rdev->irq.pflip[0]))
6443                                        radeon_crtc_handle_vblank(rdev, 0);
6444                                rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6445                                DRM_DEBUG("IH: D1 vblank\n");
6446
6447                                break;
6448                        case 1: /* D1 vline */
6449                                if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
6450                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6451
6452                                rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6453                                DRM_DEBUG("IH: D1 vline\n");
6454
6455                                break;
6456                        default:
6457                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6458                                break;
6459                        }
6460                        break;
6461                case 2: /* D2 vblank/vline */
6462                        switch (src_data) {
6463                        case 0: /* D2 vblank */
6464                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
6465                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6466
6467                                if (rdev->irq.crtc_vblank_int[1]) {
6468                                        drm_handle_vblank(rdev->ddev, 1);
6469                                        rdev->pm.vblank_sync = true;
6470                                        wake_up(&rdev->irq.vblank_queue);
6471                                }
6472                                if (atomic_read(&rdev->irq.pflip[1]))
6473                                        radeon_crtc_handle_vblank(rdev, 1);
6474                                rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6475                                DRM_DEBUG("IH: D2 vblank\n");
6476
6477                                break;
6478                        case 1: /* D2 vline */
6479                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
6480                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6481
6482                                rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6483                                DRM_DEBUG("IH: D2 vline\n");
6484
6485                                break;
6486                        default:
6487                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6488                                break;
6489                        }
6490                        break;
6491                case 3: /* D3 vblank/vline */
6492                        switch (src_data) {
6493                        case 0: /* D3 vblank */
6494                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
6495                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6496
6497                                if (rdev->irq.crtc_vblank_int[2]) {
6498                                        drm_handle_vblank(rdev->ddev, 2);
6499                                        rdev->pm.vblank_sync = true;
6500                                        wake_up(&rdev->irq.vblank_queue);
6501                                }
6502                                if (atomic_read(&rdev->irq.pflip[2]))
6503                                        radeon_crtc_handle_vblank(rdev, 2);
6504                                rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6505                                DRM_DEBUG("IH: D3 vblank\n");
6506
6507                                break;
6508                        case 1: /* D3 vline */
6509                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
6510                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6511
6512                                rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6513                                DRM_DEBUG("IH: D3 vline\n");
6514
6515                                break;
6516                        default:
6517                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6518                                break;
6519                        }
6520                        break;
6521                case 4: /* D4 vblank/vline */
6522                        switch (src_data) {
6523                        case 0: /* D4 vblank */
6524                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
6525                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6526
6527                                if (rdev->irq.crtc_vblank_int[3]) {
6528                                        drm_handle_vblank(rdev->ddev, 3);
6529                                        rdev->pm.vblank_sync = true;
6530                                        wake_up(&rdev->irq.vblank_queue);
6531                                }
6532                                if (atomic_read(&rdev->irq.pflip[3]))
6533                                        radeon_crtc_handle_vblank(rdev, 3);
6534                                rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6535                                DRM_DEBUG("IH: D4 vblank\n");
6536
6537                                break;
6538                        case 1: /* D4 vline */
6539                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
6540                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6541
6542                                rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6543                                DRM_DEBUG("IH: D4 vline\n");
6544
6545                                break;
6546                        default:
6547                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6548                                break;
6549                        }
6550                        break;
6551                case 5: /* D5 vblank/vline */
6552                        switch (src_data) {
6553                        case 0: /* D5 vblank */
6554                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
6555                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6556
6557                                if (rdev->irq.crtc_vblank_int[4]) {
6558                                        drm_handle_vblank(rdev->ddev, 4);
6559                                        rdev->pm.vblank_sync = true;
6560                                        wake_up(&rdev->irq.vblank_queue);
6561                                }
6562                                if (atomic_read(&rdev->irq.pflip[4]))
6563                                        radeon_crtc_handle_vblank(rdev, 4);
6564                                rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6565                                DRM_DEBUG("IH: D5 vblank\n");
6566
6567                                break;
6568                        case 1: /* D5 vline */
6569                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
6570                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6571
6572                                rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6573                                DRM_DEBUG("IH: D5 vline\n");
6574
6575                                break;
6576                        default:
6577                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6578                                break;
6579                        }
6580                        break;
6581                case 6: /* D6 vblank/vline */
6582                        switch (src_data) {
6583                        case 0: /* D6 vblank */
6584                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
6585                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6586
6587                                if (rdev->irq.crtc_vblank_int[5]) {
6588                                        drm_handle_vblank(rdev->ddev, 5);
6589                                        rdev->pm.vblank_sync = true;
6590                                        wake_up(&rdev->irq.vblank_queue);
6591                                }
6592                                if (atomic_read(&rdev->irq.pflip[5]))
6593                                        radeon_crtc_handle_vblank(rdev, 5);
6594                                rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6595                                DRM_DEBUG("IH: D6 vblank\n");
6596
6597                                break;
6598                        case 1: /* D6 vline */
6599                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
6600                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6601
6602                                rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6603                                DRM_DEBUG("IH: D6 vline\n");
6604
6605                                break;
6606                        default:
6607                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6608                                break;
6609                        }
6610                        break;
6611                case 8: /* D1 page flip */
6612                case 10: /* D2 page flip */
6613                case 12: /* D3 page flip */
6614                case 14: /* D4 page flip */
6615                case 16: /* D5 page flip */
6616                case 18: /* D6 page flip */
6617                        DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6618                        if (radeon_use_pflipirq > 0)
6619                                radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6620                        break;
6621                case 42: /* HPD hotplug */
6622                        switch (src_data) {
6623                        case 0:
6624                                if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
6625                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6626
6627                                rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6628                                queue_hotplug = true;
6629                                DRM_DEBUG("IH: HPD1\n");
6630
6631                                break;
6632                        case 1:
6633                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
6634                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6635
6636                                rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6637                                queue_hotplug = true;
6638                                DRM_DEBUG("IH: HPD2\n");
6639
6640                                break;
6641                        case 2:
6642                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
6643                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6644
6645                                rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6646                                queue_hotplug = true;
6647                                DRM_DEBUG("IH: HPD3\n");
6648
6649                                break;
6650                        case 3:
6651                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
6652                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6653
6654                                rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6655                                queue_hotplug = true;
6656                                DRM_DEBUG("IH: HPD4\n");
6657
6658                                break;
6659                        case 4:
6660                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
6661                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6662
6663                                rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6664                                queue_hotplug = true;
6665                                DRM_DEBUG("IH: HPD5\n");
6666
6667                                break;
6668                        case 5:
6669                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
6670                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6671
6672                                rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6673                                queue_hotplug = true;
6674                                DRM_DEBUG("IH: HPD6\n");
6675
6676                                break;
6677                        case 6:
6678                                if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
6679                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6680
6681                                rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
6682                                queue_dp = true;
6683                                DRM_DEBUG("IH: HPD_RX 1\n");
6684
6685                                break;
6686                        case 7:
6687                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
6688                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6689
6690                                rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
6691                                queue_dp = true;
6692                                DRM_DEBUG("IH: HPD_RX 2\n");
6693
6694                                break;
6695                        case 8:
6696                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
6697                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6698
6699                                rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
6700                                queue_dp = true;
6701                                DRM_DEBUG("IH: HPD_RX 3\n");
6702
6703                                break;
6704                        case 9:
6705                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
6706                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6707
6708                                rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
6709                                queue_dp = true;
6710                                DRM_DEBUG("IH: HPD_RX 4\n");
6711
6712                                break;
6713                        case 10:
6714                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
6715                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6716
6717                                rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
6718                                queue_dp = true;
6719                                DRM_DEBUG("IH: HPD_RX 5\n");
6720
6721                                break;
6722                        case 11:
6723                                if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
6724                                        DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6725
6726                                rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
6727                                queue_dp = true;
6728                                DRM_DEBUG("IH: HPD_RX 6\n");
6729
6730                                break;
6731                        default:
6732                                DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6733                                break;
6734                        }
6735                        break;
6736                case 96:
6737                        DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6738                        WREG32(SRBM_INT_ACK, 0x1);
6739                        break;
6740                case 124: /* UVD */
6741                        DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6742                        radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6743                        break;
6744                case 146:
6745                case 147:
6746                        addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6747                        status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6748                        /* reset addr and status */
6749                        WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6750                        if (addr == 0x0 && status == 0x0)
6751                                break;
6752                        dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6753                        dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
6754                                addr);
6755                        dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6756                                status);
6757                        si_vm_decode_fault(rdev, status, addr);
6758                        break;
6759                case 176: /* RINGID0 CP_INT */
6760                        radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6761                        break;
6762                case 177: /* RINGID1 CP_INT */
6763                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6764                        break;
6765                case 178: /* RINGID2 CP_INT */
6766                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6767                        break;
6768                case 181: /* CP EOP event */
6769                        DRM_DEBUG("IH: CP EOP\n");
6770                        switch (ring_id) {
6771                        case 0:
6772                                radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6773                                break;
6774                        case 1:
6775                                radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6776                                break;
6777                        case 2:
6778                                radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6779                                break;
6780                        }
6781                        break;
6782                case 224: /* DMA trap event */
6783                        DRM_DEBUG("IH: DMA trap\n");
6784                        radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6785                        break;
6786                case 230: /* thermal low to high */
6787                        DRM_DEBUG("IH: thermal low to high\n");
6788                        rdev->pm.dpm.thermal.high_to_low = false;
6789                        queue_thermal = true;
6790                        break;
6791                case 231: /* thermal high to low */
6792                        DRM_DEBUG("IH: thermal high to low\n");
6793                        rdev->pm.dpm.thermal.high_to_low = true;
6794                        queue_thermal = true;
6795                        break;
6796                case 233: /* GUI IDLE */
6797                        DRM_DEBUG("IH: GUI idle\n");
6798                        break;
6799                case 244: /* DMA trap event */
6800                        DRM_DEBUG("IH: DMA1 trap\n");
6801                        radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6802                        break;
6803                default:
6804                        DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6805                        break;
6806                }
6807
6808                /* wptr/rptr are in bytes! */
6809                rptr += 16;
6810                rptr &= rdev->ih.ptr_mask;
6811                WREG32(IH_RB_RPTR, rptr);
6812        }
6813        if (queue_dp)
6814                schedule_work(&rdev->dp_work);
6815        if (queue_hotplug)
6816                schedule_delayed_work(&rdev->hotplug_work, 0);
6817        if (queue_thermal && rdev->pm.dpm_enabled)
6818                schedule_work(&rdev->pm.dpm.thermal.work);
6819        rdev->ih.rptr = rptr;
6820        atomic_set(&rdev->ih.lock, 0);
6821
6822        /* make sure wptr hasn't changed while processing */
6823        wptr = si_get_ih_wptr(rdev);
6824        if (wptr != rptr)
6825                goto restart_ih;
6826
6827        return IRQ_HANDLED;
6828}
6829
6830/*
6831 * startup/shutdown callbacks
6832 */
6833static void si_uvd_init(struct radeon_device *rdev)
6834{
6835        int r;
6836
6837        if (!rdev->has_uvd)
6838                return;
6839
6840        r = radeon_uvd_init(rdev);
6841        if (r) {
6842                dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
6843                /*
6844                 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
6845                 * to early fails uvd_v2_2_resume() and thus nothing happens
6846                 * there. So it is pointless to try to go through that code
6847                 * hence why we disable uvd here.
6848                 */
6849                rdev->has_uvd = 0;
6850                return;
6851        }
6852        rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
6853        r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
6854}
6855
6856static void si_uvd_start(struct radeon_device *rdev)
6857{
6858        int r;
6859
6860        if (!rdev->has_uvd)
6861                return;
6862
6863        r = uvd_v2_2_resume(rdev);
6864        if (r) {
6865                dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
6866                goto error;
6867        }
6868        r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
6869        if (r) {
6870                dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
6871                goto error;
6872        }
6873        return;
6874
6875error:
6876        rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6877}
6878
6879static void si_uvd_resume(struct radeon_device *rdev)
6880{
6881        struct radeon_ring *ring;
6882        int r;
6883
6884        if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
6885                return;
6886
6887        ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6888        r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2);
6889        if (r) {
6890                dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
6891                return;
6892        }
6893        r = uvd_v1_0_init(rdev);
6894        if (r) {
6895                dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
6896                return;
6897        }
6898}
6899
6900static void si_vce_init(struct radeon_device *rdev)
6901{
6902        int r;
6903
6904        if (!rdev->has_vce)
6905                return;
6906
6907        r = radeon_vce_init(rdev);
6908        if (r) {
6909                dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
6910                /*
6911                 * At this point rdev->vce.vcpu_bo is NULL which trickles down
6912                 * to early fails si_vce_start() and thus nothing happens
6913                 * there. So it is pointless to try to go through that code
6914                 * hence why we disable vce here.
6915                 */
6916                rdev->has_vce = 0;
6917                return;
6918        }
6919        rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
6920        r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
6921        rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
6922        r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
6923}
6924
6925static void si_vce_start(struct radeon_device *rdev)
6926{
6927        int r;
6928
6929        if (!rdev->has_vce)
6930                return;
6931
6932        r = radeon_vce_resume(rdev);
6933        if (r) {
6934                dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6935                goto error;
6936        }
6937        r = vce_v1_0_resume(rdev);
6938        if (r) {
6939                dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6940                goto error;
6941        }
6942        r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
6943        if (r) {
6944                dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
6945                goto error;
6946        }
6947        r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
6948        if (r) {
6949                dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
6950                goto error;
6951        }
6952        return;
6953
6954error:
6955        rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6956        rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6957}
6958
6959static void si_vce_resume(struct radeon_device *rdev)
6960{
6961        struct radeon_ring *ring;
6962        int r;
6963
6964        if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
6965                return;
6966
6967        ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
6968        r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6969        if (r) {
6970                dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6971                return;
6972        }
6973        ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
6974        r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6975        if (r) {
6976                dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6977                return;
6978        }
6979        r = vce_v1_0_init(rdev);
6980        if (r) {
6981                dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
6982                return;
6983        }
6984}
6985
6986static int si_startup(struct radeon_device *rdev)
6987{
6988        struct radeon_ring *ring;
6989        int r;
6990
6991        /* enable pcie gen2/3 link */
6992        si_pcie_gen3_enable(rdev);
6993        /* enable aspm */
6994        si_program_aspm(rdev);
6995
6996        /* scratch needs to be initialized before MC */
6997        r = r600_vram_scratch_init(rdev);
6998        if (r)
6999                return r;
7000

7001        si_mc_program(rdev);
7002
7003        if (!rdev->pm.dpm_enabled) {
7004                r = si_mc_load_microcode(rdev);
7005                if (r) {
7006                        DRM_ERROR("Failed to load MC firmware!\n");
7007                        return r;
7008                }
7009        }
7010
7011        r = si_pcie_gart_enable(rdev);
7012        if (r)
7013                return r;
7014        si_gpu_init(rdev);
7015
7016        /* allocate rlc buffers */
7017        if (rdev->family == CHIP_VERDE) {
7018                rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
7019                rdev->rlc.reg_list_size =
7020                        (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
7021        }
7022        rdev->rlc.cs_data = si_cs_data;
7023        r = sumo_rlc_init(rdev);
7024        if (r) {
7025                DRM_ERROR("Failed to init rlc BOs!\n");
7026                return r;
7027        }
7028
7029        /* allocate wb buffer */
7030        r = radeon_wb_init(rdev);
7031        if (r)
7032                return r;
7033
7034        r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7035        if (r) {
7036                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7037                return r;
7038        }
7039
7040        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7041        if (r) {
7042                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7043                return r;
7044        }
7045
7046        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7047        if (r) {
7048                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7049                return r;
7050        }
7051
7052        r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7053        if (r) {
7054                dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7055                return r;
7056        }
7057
7058        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7059        if (r) {
7060                dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7061                return r;
7062        }
7063
7064        si_uvd_start(rdev);
7065        si_vce_start(rdev);
7066
7067        /* Enable IRQ */
7068        if (!rdev->irq.installed) {
7069                r = radeon_irq_kms_init(rdev);
7070                if (r)
7071                        return r;
7072        }
7073
7074        r = si_irq_init(rdev);
7075        if (r) {
7076                DRM_ERROR("radeon: IH init failed (%d).\n", r);
7077                radeon_irq_kms_fini(rdev);
7078                return r;
7079        }
7080        si_irq_set(rdev);
7081
7082        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7083        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7084                             RADEON_CP_PACKET2);
7085        if (r)
7086                return r;
7087
7088        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7089        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7090                             RADEON_CP_PACKET2);
7091        if (r)
7092                return r;
7093
7094        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7095        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7096                             RADEON_CP_PACKET2);
7097        if (r)
7098                return r;
7099
7100        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7101        r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7102                             DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
7103        if (r)
7104                return r;
7105
7106        ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7107        r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7108                             DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
7109        if (r)
7110                return r;
7111
7112        r = si_cp_load_microcode(rdev);
7113        if (r)
7114                return r;
7115        r = si_cp_resume(rdev);
7116        if (r)
7117                return r;
7118
7119        r = cayman_dma_resume(rdev);
7120        if (r)
7121                return r;
7122
7123        si_uvd_resume(rdev);
7124        si_vce_resume(rdev);
7125
7126        r = radeon_ib_pool_init(rdev);
7127        if (r) {
7128                dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7129                return r;
7130        }
7131
7132        r = radeon_vm_manager_init(rdev);
7133        if (r) {
7134                dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7135                return r;
7136        }
7137
7138        r = radeon_audio_init(rdev);
7139        if (r)
7140                return r;
7141
7142        return 0;
7143}
7144
7145int si_resume(struct radeon_device *rdev)
7146{
7147        int r;
7148
7149        /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
7150         * posting will perform necessary task to bring back GPU into good
7151         * shape.
7152         */
7153        /* post card */
7154        atom_asic_init(rdev->mode_info.atom_context);
7155
7156        /* init golden registers */
7157        si_init_golden_registers(rdev);
7158
7159        if (rdev->pm.pm_method == PM_METHOD_DPM)
7160                radeon_pm_resume(rdev);
7161
7162        rdev->accel_working = true;
7163        r = si_startup(rdev);
7164        if (r) {
7165                DRM_ERROR("si startup failed on resume\n");
7166                rdev->accel_working = false;
7167                return r;
7168        }
7169
7170        return r;
7171
7172}
7173
7174int si_suspend(struct radeon_device *rdev)
7175{
7176        radeon_pm_suspend(rdev);
7177        radeon_audio_fini(rdev);
7178        radeon_vm_manager_fini(rdev);
7179        si_cp_enable(rdev, false);
7180        cayman_dma_stop(rdev);
7181        if (rdev->has_uvd) {
7182                uvd_v1_0_fini(rdev);
7183                radeon_uvd_suspend(rdev);
7184        }
7185        if (rdev->has_vce)
7186                radeon_vce_suspend(rdev);
7187        si_fini_pg(rdev);
7188        si_fini_cg(rdev);
7189        si_irq_suspend(rdev);
7190        radeon_wb_disable(rdev);
7191        si_pcie_gart_disable(rdev);
7192        return 0;
7193}
7194
7195/* Plan is to move initialization in that function and use
7196 * helper function so that radeon_device_init pretty much
7197 * do nothing more than calling asic specific function. This
7198 * should also allow to remove a bunch of callback function
7199 * like vram_info.
7200 */
7201int si_init(struct radeon_device *rdev)
7202{
7203        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7204        int r;
7205
7206        /* Read BIOS */
7207        if (!radeon_get_bios(rdev)) {
7208                if (ASIC_IS_AVIVO(rdev))
7209                        return -EINVAL;
7210        }
7211        /* Must be an ATOMBIOS */
7212        if (!rdev->is_atom_bios) {
7213                dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7214                return -EINVAL;
7215        }
7216        r = radeon_atombios_init(rdev);
7217        if (r)
7218                return r;
7219
7220        /* Post card if necessary */
7221        if (!radeon_card_posted(rdev)) {
7222                if (!rdev->bios) {
7223                        dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7224                        return -EINVAL;
7225                }
7226                DRM_INFO("GPU not posted. posting now...\n");
7227                atom_asic_init(rdev->mode_info.atom_context);
7228        }
7229        /* init golden registers */
7230        si_init_golden_registers(rdev);
7231        /* Initialize scratch registers */
7232        si_scratch_init(rdev);
7233        /* Initialize surface registers */
7234        radeon_surface_init(rdev);
7235        /* Initialize clocks */
7236        radeon_get_clock_info(rdev->ddev);
7237
7238        /* Fence driver */
7239        r = radeon_fence_driver_init(rdev);
7240        if (r)
7241                return r;
7242
7243        /* initialize memory controller */
7244        r = si_mc_init(rdev);
7245        if (r)
7246                return r;
7247        /* Memory manager */
7248        r = radeon_bo_init(rdev);
7249        if (r)
7250                return r;
7251
7252        if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7253            !rdev->rlc_fw || !rdev->mc_fw) {
7254                r = si_init_microcode(rdev);
7255                if (r) {
7256                        DRM_ERROR("Failed to load firmware!\n");
7257                        return r;
7258                }
7259        }
7260
7261        /* Initialize power management */
7262        radeon_pm_init(rdev);
7263
7264        ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7265        ring->ring_obj = NULL;
7266        r600_ring_init(rdev, ring, 1024 * 1024);
7267
7268        ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7269        ring->ring_obj = NULL;
7270        r600_ring_init(rdev, ring, 1024 * 1024);
7271
7272        ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7273        ring->ring_obj = NULL;
7274        r600_ring_init(rdev, ring, 1024 * 1024);
7275
7276        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7277        ring->ring_obj = NULL;
7278        r600_ring_init(rdev, ring, 64 * 1024);
7279
7280        ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7281        ring->ring_obj = NULL;
7282        r600_ring_init(rdev, ring, 64 * 1024);
7283
7284        si_uvd_init(rdev);
7285        si_vce_init(rdev);
7286
7287        rdev->ih.ring_obj = NULL;
7288        r600_ih_ring_init(rdev, 64 * 1024);
7289
7290        r = r600_pcie_gart_init(rdev);
7291        if (r)
7292                return r;
7293
7294        rdev->accel_working = true;
7295        r = si_startup(rdev);
7296        if (r) {
7297                dev_err(rdev->dev, "disabling GPU acceleration\n");
7298                si_cp_fini(rdev);
7299                cayman_dma_fini(rdev);
7300                si_irq_fini(rdev);
7301                sumo_rlc_fini(rdev);
7302                radeon_wb_fini(rdev);
7303                radeon_ib_pool_fini(rdev);
7304                radeon_vm_manager_fini(rdev);
7305                radeon_irq_kms_fini(rdev);
7306                si_pcie_gart_fini(rdev);
7307                rdev->accel_working = false;
7308        }
7309
7310        /* Don't start up if the MC ucode is missing.
7311         * The default clocks and voltages before the MC ucode
7312         * is loaded are not suffient for advanced operations.
7313         */
7314        if (!rdev->mc_fw) {
7315                DRM_ERROR("radeon: MC ucode required for NI+.\n");
7316                return -EINVAL;
7317        }
7318
7319        return 0;
7320}
7321
7322void si_fini(struct radeon_device *rdev)
7323{
7324        radeon_pm_fini(rdev);
7325        si_cp_fini(rdev);
7326        cayman_dma_fini(rdev);
7327        si_fini_pg(rdev);
7328        si_fini_cg(rdev);
7329        si_irq_fini(rdev);
7330        sumo_rlc_fini(rdev);
7331        radeon_wb_fini(rdev);
7332        radeon_vm_manager_fini(rdev);
7333        radeon_ib_pool_fini(rdev);
7334        radeon_irq_kms_fini(rdev);
7335        if (rdev->has_uvd) {
7336                uvd_v1_0_fini(rdev);
7337                radeon_uvd_fini(rdev);
7338        }
7339        if (rdev->has_vce)
7340                radeon_vce_fini(rdev);
7341        si_pcie_gart_fini(rdev);
7342        r600_vram_scratch_fini(rdev);
7343        radeon_gem_fini(rdev);
7344        radeon_fence_driver_fini(rdev);
7345        radeon_bo_fini(rdev);
7346        radeon_atombios_fini(rdev);
7347        kfree(rdev->bios);
7348        rdev->bios = NULL;
7349}
7350
7351/**
7352 * si_get_gpu_clock_counter - return GPU clock counter snapshot
7353 *
7354 * @rdev: radeon_device pointer
7355 *
7356 * Fetches a GPU clock counter snapshot (SI).
7357 * Returns the 64 bit clock counter snapshot.
7358 */
7359uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
7360{
7361        uint64_t clock;
7362
7363        mutex_lock(&rdev->gpu_clock_mutex);
7364        WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
7365        clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
7366                ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
7367        mutex_unlock(&rdev->gpu_clock_mutex);
7368        return clock;
7369}
7370
7371int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
7372{
7373        unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
7374        int r;
7375
7376        /* bypass vclk and dclk with bclk */
7377        WREG32_P(CG_UPLL_FUNC_CNTL_2,
7378                VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
7379                ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7380
7381        /* put PLL in bypass mode */
7382        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7383
7384        if (!vclk || !dclk) {
7385                /* keep the Bypass mode */
7386                return 0;
7387        }
7388
7389        r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7390                                          16384, 0x03FFFFFF, 0, 128, 5,
7391                                          &fb_div, &vclk_div, &dclk_div);
7392        if (r)
7393                return r;
7394
7395        /* set RESET_ANTI_MUX to 0 */
7396        WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7397
7398        /* set VCO_MODE to 1 */
7399        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7400
7401        /* disable sleep mode */
7402        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7403
7404        /* deassert UPLL_RESET */
7405        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7406
7407        mdelay(1);
7408
7409        r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7410        if (r)
7411                return r;
7412
7413        /* assert UPLL_RESET again */
7414        WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7415
7416        /* disable spread spectrum. */
7417        WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7418
7419        /* set feedback divider */
7420        WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
7421
7422        /* set ref divider to 0 */
7423        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7424
7425        if (fb_div < 307200)
7426                WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7427        else
7428                WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7429
7430        /* set PDIV_A and PDIV_B */
7431        WREG32_P(CG_UPLL_FUNC_CNTL_2,
7432                UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
7433                ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7434
7435        /* give the PLL some time to settle */
7436        mdelay(15);
7437
7438        /* deassert PLL_RESET */
7439        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7440
7441        mdelay(15);
7442
7443        /* switch from bypass mode to normal mode */
7444        WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7445
7446        r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7447        if (r)
7448                return r;
7449
7450        /* switch VCLK and DCLK selection */
7451        WREG32_P(CG_UPLL_FUNC_CNTL_2,
7452                VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7453                ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7454
7455        mdelay(100);
7456
7457        return 0;
7458}
7459
7460static void si_pcie_gen3_enable(struct radeon_device *rdev)
7461{
7462        struct pci_dev *root = rdev->pdev->bus->self;
7463        int bridge_pos, gpu_pos;
7464        u32 speed_cntl, mask, current_data_rate;
7465        int ret, i;
7466        u16 tmp16;
7467
7468        if (pci_is_root_bus(rdev->pdev->bus))
7469                return;
7470
7471        if (radeon_pcie_gen2 == 0)
7472                return;
7473
7474        if (rdev->flags & RADEON_IS_IGP)
7475                return;
7476
7477        if (!(rdev->flags & RADEON_IS_PCIE))
7478                return;
7479
7480        ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
7481        if (ret != 0)
7482                return;
7483
7484        if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
7485                return;
7486
7487        speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7488        current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7489                LC_CURRENT_DATA_RATE_SHIFT;
7490        if (mask & DRM_PCIE_SPEED_80) {
7491                if (current_data_rate == 2) {
7492                        DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7493                        return;
7494                }
7495                DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7496        } else if (mask & DRM_PCIE_SPEED_50) {
7497                if (current_data_rate == 1) {
7498                        DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7499                        return;
7500                }
7501                DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7502        }
7503
7504        bridge_pos = pci_pcie_cap(root);
7505        if (!bridge_pos)
7506                return;
7507
7508        gpu_pos = pci_pcie_cap(rdev->pdev);
7509        if (!gpu_pos)
7510                return;
7511
7512        if (mask & DRM_PCIE_SPEED_80) {
7513                /* re-try equalization if gen3 is not already enabled */
7514                if (current_data_rate != 2) {
7515                        u16 bridge_cfg, gpu_cfg;
7516                        u16 bridge_cfg2, gpu_cfg2;
7517                        u32 max_lw, current_lw, tmp;
7518
7519                        pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7520                        pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7521
7522                        tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7523                        pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7524
7525                        tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7526                        pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7527
7528                        tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7529                        max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7530                        current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7531
7532                        if (current_lw < max_lw) {
7533                                tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7534                                if (tmp & LC_RENEGOTIATION_SUPPORT) {
7535                                        tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7536                                        tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7537                                        tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7538                                        WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7539                                }
7540                        }
7541
7542                        for (i = 0; i < 10; i++) {
7543                                /* check status */
7544                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7545                                if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7546                                        break;
7547
7548                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7549                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7550
7551                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7552                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7553
7554                                tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7555                                tmp |= LC_SET_QUIESCE;
7556                                WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7557
7558                                tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7559                                tmp |= LC_REDO_EQ;
7560                                WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7561
7562                                mdelay(100);
7563
7564                                /* linkctl */
7565                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7566                                tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7567                                tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7568                                pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7569
7570                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7571                                tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7572                                tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7573                                pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7574
7575                                /* linkctl2 */
7576                                pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7577                                tmp16 &= ~((1 << 4) | (7 << 9));
7578                                tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7579                                pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7580
7581                                pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7582                                tmp16 &= ~((1 << 4) | (7 << 9));
7583                                tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7584                                pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7585
7586                                tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7587                                tmp &= ~LC_SET_QUIESCE;
7588                                WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7589                        }
7590                }
7591        }
7592
7593        /* set the link speed */
7594        speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7595        speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7596        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7597
7598        pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7599        tmp16 &= ~0xf;
7600        if (mask & DRM_PCIE_SPEED_80)
7601                tmp16 |= 3; /* gen3 */
7602        else if (mask & DRM_PCIE_SPEED_50)
7603                tmp16 |= 2; /* gen2 */
7604        else
7605                tmp16 |= 1; /* gen1 */
7606        pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7607
7608        speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7609        speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7610        WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7611
7612        for (i = 0; i < rdev->usec_timeout; i++) {
7613                speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7614                if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7615                        break;
7616                udelay(1);
7617        }
7618}
7619
7620static void si_program_aspm(struct radeon_device *rdev)
7621{
7622        u32 data, orig;
7623        bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7624        bool disable_clkreq = false;
7625
7626        if (radeon_aspm == 0)
7627                return;
7628
7629        if (!(rdev->flags & RADEON_IS_PCIE))
7630                return;
7631
7632        orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7633        data &= ~LC_XMIT_N_FTS_MASK;
7634        data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7635        if (orig != data)
7636                WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7637
7638        orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7639        data |= LC_GO_TO_RECOVERY;
7640        if (orig != data)
7641                WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7642
7643        orig = data = RREG32_PCIE(PCIE_P_CNTL);
7644        data |= P_IGNORE_EDB_ERR;
7645        if (orig != data)
7646                WREG32_PCIE(PCIE_P_CNTL, data);
7647
7648        orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7649        data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7650        data |= LC_PMI_TO_L1_DIS;
7651        if (!disable_l0s)
7652                data |= LC_L0S_INACTIVITY(7);
7653
7654        if (!disable_l1) {
7655                data |= LC_L1_INACTIVITY(7);
7656                data &= ~LC_PMI_TO_L1_DIS;
7657                if (orig != data)
7658                        WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7659
7660                if (!disable_plloff_in_l1) {
7661                        bool clk_req_support;
7662
7663                        orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7664                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7665                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7666                        if (orig != data)
7667                                WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7668
7669                        orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7670                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7671                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7672                        if (orig != data)
7673                                WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7674
7675                        orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7676                        data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7677                        data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7678                        if (orig != data)
7679                                WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7680
7681                        orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7682                        data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7683                        data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7684                        if (orig != data)
7685                                WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7686
7687                        if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7688                                orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7689                                data &= ~PLL_RAMP_UP_TIME_0_MASK;
7690                                if (orig != data)
7691                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7692
7693                                orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7694                                data &= ~PLL_RAMP_UP_TIME_1_MASK;
7695                                if (orig != data)
7696                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7697
7698                                orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7699                                data &= ~PLL_RAMP_UP_TIME_2_MASK;
7700                                if (orig != data)
7701                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7702
7703                                orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7704                                data &= ~PLL_RAMP_UP_TIME_3_MASK;
7705                                if (orig != data)
7706                                        WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7707
7708                                orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7709                                data &= ~PLL_RAMP_UP_TIME_0_MASK;
7710                                if (orig != data)
7711                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7712
7713                                orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7714                                data &= ~PLL_RAMP_UP_TIME_1_MASK;
7715                                if (orig != data)
7716                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7717
7718                                orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7719                                data &= ~PLL_RAMP_UP_TIME_2_MASK;
7720                                if (orig != data)
7721                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7722
7723                                orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7724                                data &= ~PLL_RAMP_UP_TIME_3_MASK;
7725                                if (orig != data)
7726                                        WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7727                        }
7728                        orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7729                        data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7730                        data |= LC_DYN_LANES_PWR_STATE(3);
7731                        if (orig != data)
7732                                WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7733
7734                        orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7735                        data &= ~LS2_EXIT_TIME_MASK;
7736                        if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7737                                data |= LS2_EXIT_TIME(5);
7738                        if (orig != data)
7739                                WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7740
7741                        orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7742                        data &= ~LS2_EXIT_TIME_MASK;
7743                        if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7744                                data |= LS2_EXIT_TIME(5);
7745                        if (orig != data)
7746                                WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7747
7748                        if (!disable_clkreq &&
7749                            !pci_is_root_bus(rdev->pdev->bus)) {
7750                                struct pci_dev *root = rdev->pdev->bus->self;
7751                                u32 lnkcap;
7752
7753                                clk_req_support = false;
7754                                pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7755                                if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7756                                        clk_req_support = true;
7757                        } else {
7758                                clk_req_support = false;
7759                        }
7760
7761                        if (clk_req_support) {
7762                                orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7763                                data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7764                                if (orig != data)
7765                                        WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7766
7767                                orig = data = RREG32(THM_CLK_CNTL);
7768                                data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7769                                data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7770                                if (orig != data)
7771                                        WREG32(THM_CLK_CNTL, data);
7772
7773                                orig = data = RREG32(MISC_CLK_CNTL);
7774                                data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7775                                data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7776                                if (orig != data)
7777                                        WREG32(MISC_CLK_CNTL, data);
7778
7779                                orig = data = RREG32(CG_CLKPIN_CNTL);
7780                                data &= ~BCLK_AS_XCLK;
7781                                if (orig != data)
7782                                        WREG32(CG_CLKPIN_CNTL, data);
7783
7784                                orig = data = RREG32(CG_CLKPIN_CNTL_2);
7785                                data &= ~FORCE_BIF_REFCLK_EN;
7786                                if (orig != data)
7787                                        WREG32(CG_CLKPIN_CNTL_2, data);
7788
7789                                orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7790                                data &= ~MPLL_CLKOUT_SEL_MASK;
7791                                data |= MPLL_CLKOUT_SEL(4);
7792                                if (orig != data)
7793                                        WREG32(MPLL_BYPASSCLK_SEL, data);
7794
7795                                orig = data = RREG32(SPLL_CNTL_MODE);
7796                                data &= ~SPLL_REFCLK_SEL_MASK;
7797                                if (orig != data)
7798                                        WREG32(SPLL_CNTL_MODE, data);
7799                        }
7800                }
7801        } else {
7802                if (orig != data)
7803                        WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7804        }
7805
7806        orig = data = RREG32_PCIE(PCIE_CNTL2);
7807        data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7808        if (orig != data)
7809                WREG32_PCIE(PCIE_CNTL2, data);
7810
7811        if (!disable_l0s) {
7812                data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7813                if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7814                        data = RREG32_PCIE(PCIE_LC_STATUS1);
7815                        if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7816                                orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7817                                data &= ~LC_L0S_INACTIVITY_MASK;
7818                                if (orig != data)
7819                                        WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7820                        }
7821                }
7822        }
7823}
7824
7825int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
7826{
7827        unsigned i;
7828
7829        /* make sure VCEPLL_CTLREQ is deasserted */
7830        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7831
7832        mdelay(10);
7833
7834        /* assert UPLL_CTLREQ */
7835        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
7836
7837        /* wait for CTLACK and CTLACK2 to get asserted */
7838        for (i = 0; i < 100; ++i) {
7839                uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7840                if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7841                        break;
7842                mdelay(10);
7843        }
7844
7845        /* deassert UPLL_CTLREQ */
7846        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7847
7848        if (i == 100) {
7849                DRM_ERROR("Timeout setting UVD clocks!\n");
7850                return -ETIMEDOUT;
7851        }
7852
7853        return 0;
7854}
7855
7856int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7857{
7858        unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7859        int r;
7860
7861        /* bypass evclk and ecclk with bclk */
7862        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7863                     EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7864                     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7865
7866        /* put PLL in bypass mode */
7867        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7868                     ~VCEPLL_BYPASS_EN_MASK);
7869
7870        if (!evclk || !ecclk) {
7871                /* keep the Bypass mode, put PLL to sleep */
7872                WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7873                             ~VCEPLL_SLEEP_MASK);
7874                return 0;
7875        }
7876
7877        r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7878                                          16384, 0x03FFFFFF, 0, 128, 5,
7879                                          &fb_div, &evclk_div, &ecclk_div);
7880        if (r)
7881                return r;
7882
7883        /* set RESET_ANTI_MUX to 0 */
7884        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7885
7886        /* set VCO_MODE to 1 */
7887        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7888                     ~VCEPLL_VCO_MODE_MASK);
7889
7890        /* toggle VCEPLL_SLEEP to 1 then back to 0 */
7891        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7892                     ~VCEPLL_SLEEP_MASK);
7893        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7894
7895        /* deassert VCEPLL_RESET */
7896        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7897
7898        mdelay(1);
7899
7900        r = si_vce_send_vcepll_ctlreq(rdev);
7901        if (r)
7902                return r;
7903
7904        /* assert VCEPLL_RESET again */
7905        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7906
7907        /* disable spread spectrum. */
7908        WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7909
7910        /* set feedback divider */
7911        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7912
7913        /* set ref divider to 0 */
7914        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7915
7916        /* set PDIV_A and PDIV_B */
7917        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7918                     VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7919                     ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7920
7921        /* give the PLL some time to settle */
7922        mdelay(15);
7923
7924        /* deassert PLL_RESET */
7925        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7926
7927        mdelay(15);
7928
7929        /* switch from bypass mode to normal mode */
7930        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7931
7932        r = si_vce_send_vcepll_ctlreq(rdev);
7933        if (r)
7934                return r;
7935
7936        /* switch VCLK and DCLK selection */
7937        WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7938                     EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7939                     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7940
7941        mdelay(100);
7942
7943        return 0;
7944}
7945