linux/drivers/gpu/drm/radeon/ni.c
<<
>>
Prefs
   1/*
   2 * Copyright 2010 Advanced Micro Devices, Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 *
  22 * Authors: Alex Deucher
  23 */
  24#include <linux/firmware.h>
  25#include <linux/slab.h>
  26#include <linux/module.h>
  27#include <drm/drmP.h>
  28#include "radeon.h"
  29#include "radeon_asic.h"
  30#include "radeon_audio.h"
  31#include <drm/radeon_drm.h>
  32#include "nid.h"
  33#include "atom.h"
  34#include "ni_reg.h"
  35#include "cayman_blit_shaders.h"
  36#include "radeon_ucode.h"
  37#include "clearstate_cayman.h"
  38
  39/*
  40 * Indirect registers accessor
  41 */
  42u32 tn_smc_rreg(struct radeon_device *rdev, u32 reg)
  43{
  44        unsigned long flags;
  45        u32 r;
  46
  47        spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  48        WREG32(TN_SMC_IND_INDEX_0, (reg));
  49        r = RREG32(TN_SMC_IND_DATA_0);
  50        spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  51        return r;
  52}
  53
  54void tn_smc_wreg(struct radeon_device *rdev, u32 reg, u32 v)
  55{
  56        unsigned long flags;
  57
  58        spin_lock_irqsave(&rdev->smc_idx_lock, flags);
  59        WREG32(TN_SMC_IND_INDEX_0, (reg));
  60        WREG32(TN_SMC_IND_DATA_0, (v));
  61        spin_unlock_irqrestore(&rdev->smc_idx_lock, flags);
  62}
  63
  64static const u32 tn_rlc_save_restore_register_list[] =
  65{
  66        0x98fc,
  67        0x98f0,
  68        0x9834,
  69        0x9838,
  70        0x9870,
  71        0x9874,
  72        0x8a14,
  73        0x8b24,
  74        0x8bcc,
  75        0x8b10,
  76        0x8c30,
  77        0x8d00,
  78        0x8d04,
  79        0x8c00,
  80        0x8c04,
  81        0x8c10,
  82        0x8c14,
  83        0x8d8c,
  84        0x8cf0,
  85        0x8e38,
  86        0x9508,
  87        0x9688,
  88        0x9608,
  89        0x960c,
  90        0x9610,
  91        0x9614,
  92        0x88c4,
  93        0x8978,
  94        0x88d4,
  95        0x900c,
  96        0x9100,
  97        0x913c,
  98        0x90e8,
  99        0x9354,
 100        0xa008,
 101        0x98f8,
 102        0x9148,
 103        0x914c,
 104        0x3f94,
 105        0x98f4,
 106        0x9b7c,
 107        0x3f8c,
 108        0x8950,
 109        0x8954,
 110        0x8a18,
 111        0x8b28,
 112        0x9144,
 113        0x3f90,
 114        0x915c,
 115        0x9160,
 116        0x9178,
 117        0x917c,
 118        0x9180,
 119        0x918c,
 120        0x9190,
 121        0x9194,
 122        0x9198,
 123        0x919c,
 124        0x91a8,
 125        0x91ac,
 126        0x91b0,
 127        0x91b4,
 128        0x91b8,
 129        0x91c4,
 130        0x91c8,
 131        0x91cc,
 132        0x91d0,
 133        0x91d4,
 134        0x91e0,
 135        0x91e4,
 136        0x91ec,
 137        0x91f0,
 138        0x91f4,
 139        0x9200,
 140        0x9204,
 141        0x929c,
 142        0x8030,
 143        0x9150,
 144        0x9a60,
 145        0x920c,
 146        0x9210,
 147        0x9228,
 148        0x922c,
 149        0x9244,
 150        0x9248,
 151        0x91e8,
 152        0x9294,
 153        0x9208,
 154        0x9224,
 155        0x9240,
 156        0x9220,
 157        0x923c,
 158        0x9258,
 159        0x9744,
 160        0xa200,
 161        0xa204,
 162        0xa208,
 163        0xa20c,
 164        0x8d58,
 165        0x9030,
 166        0x9034,
 167        0x9038,
 168        0x903c,
 169        0x9040,
 170        0x9654,
 171        0x897c,
 172        0xa210,
 173        0xa214,
 174        0x9868,
 175        0xa02c,
 176        0x9664,
 177        0x9698,
 178        0x949c,
 179        0x8e10,
 180        0x8e18,
 181        0x8c50,
 182        0x8c58,
 183        0x8c60,
 184        0x8c68,
 185        0x89b4,
 186        0x9830,
 187        0x802c,
 188};
 189
 190extern bool evergreen_is_display_hung(struct radeon_device *rdev);
 191extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 192extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
 193extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
 194extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
 195extern void evergreen_mc_program(struct radeon_device *rdev);
 196extern void evergreen_irq_suspend(struct radeon_device *rdev);
 197extern int evergreen_mc_init(struct radeon_device *rdev);
 198extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
 199extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 200extern void evergreen_program_aspm(struct radeon_device *rdev);
 201extern void sumo_rlc_fini(struct radeon_device *rdev);
 202extern int sumo_rlc_init(struct radeon_device *rdev);
 203extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
 204
 205/* Firmware Names */
 206MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
 207MODULE_FIRMWARE("radeon/BARTS_me.bin");
 208MODULE_FIRMWARE("radeon/BARTS_mc.bin");
 209MODULE_FIRMWARE("radeon/BARTS_smc.bin");
 210MODULE_FIRMWARE("radeon/BTC_rlc.bin");
 211MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
 212MODULE_FIRMWARE("radeon/TURKS_me.bin");
 213MODULE_FIRMWARE("radeon/TURKS_mc.bin");
 214MODULE_FIRMWARE("radeon/TURKS_smc.bin");
 215MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
 216MODULE_FIRMWARE("radeon/CAICOS_me.bin");
 217MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
 218MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
 219MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
 220MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
 221MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
 222MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
 223MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
 224MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
 225MODULE_FIRMWARE("radeon/ARUBA_me.bin");
 226MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
 227
 228
 229static const u32 cayman_golden_registers2[] =
 230{
 231        0x3e5c, 0xffffffff, 0x00000000,
 232        0x3e48, 0xffffffff, 0x00000000,
 233        0x3e4c, 0xffffffff, 0x00000000,
 234        0x3e64, 0xffffffff, 0x00000000,
 235        0x3e50, 0xffffffff, 0x00000000,
 236        0x3e60, 0xffffffff, 0x00000000
 237};
 238
 239static const u32 cayman_golden_registers[] =
 240{
 241        0x5eb4, 0xffffffff, 0x00000002,
 242        0x5e78, 0x8f311ff1, 0x001000f0,
 243        0x3f90, 0xffff0000, 0xff000000,
 244        0x9148, 0xffff0000, 0xff000000,
 245        0x3f94, 0xffff0000, 0xff000000,
 246        0x914c, 0xffff0000, 0xff000000,
 247        0xc78, 0x00000080, 0x00000080,
 248        0xbd4, 0x70073777, 0x00011003,
 249        0xd02c, 0xbfffff1f, 0x08421000,
 250        0xd0b8, 0x73773777, 0x02011003,
 251        0x5bc0, 0x00200000, 0x50100000,
 252        0x98f8, 0x33773777, 0x02011003,
 253        0x98fc, 0xffffffff, 0x76541032,
 254        0x7030, 0x31000311, 0x00000011,
 255        0x2f48, 0x33773777, 0x42010001,
 256        0x6b28, 0x00000010, 0x00000012,
 257        0x7728, 0x00000010, 0x00000012,
 258        0x10328, 0x00000010, 0x00000012,
 259        0x10f28, 0x00000010, 0x00000012,
 260        0x11b28, 0x00000010, 0x00000012,
 261        0x12728, 0x00000010, 0x00000012,
 262        0x240c, 0x000007ff, 0x00000000,
 263        0x8a14, 0xf000001f, 0x00000007,
 264        0x8b24, 0x3fff3fff, 0x00ff0fff,
 265        0x8b10, 0x0000ff0f, 0x00000000,
 266        0x28a4c, 0x07ffffff, 0x06000000,
 267        0x10c, 0x00000001, 0x00010003,
 268        0xa02c, 0xffffffff, 0x0000009b,
 269        0x913c, 0x0000010f, 0x01000100,
 270        0x8c04, 0xf8ff00ff, 0x40600060,
 271        0x28350, 0x00000f01, 0x00000000,
 272        0x9508, 0x3700001f, 0x00000002,
 273        0x960c, 0xffffffff, 0x54763210,
 274        0x88c4, 0x001f3ae3, 0x00000082,
 275        0x88d0, 0xffffffff, 0x0f40df40,
 276        0x88d4, 0x0000001f, 0x00000010,
 277        0x8974, 0xffffffff, 0x00000000
 278};
 279
 280static const u32 dvst_golden_registers2[] =
 281{
 282        0x8f8, 0xffffffff, 0,
 283        0x8fc, 0x00380000, 0,
 284        0x8f8, 0xffffffff, 1,
 285        0x8fc, 0x0e000000, 0
 286};
 287
 288static const u32 dvst_golden_registers[] =
 289{
 290        0x690, 0x3fff3fff, 0x20c00033,
 291        0x918c, 0x0fff0fff, 0x00010006,
 292        0x91a8, 0x0fff0fff, 0x00010006,
 293        0x9150, 0xffffdfff, 0x6e944040,
 294        0x917c, 0x0fff0fff, 0x00030002,
 295        0x9198, 0x0fff0fff, 0x00030002,
 296        0x915c, 0x0fff0fff, 0x00010000,
 297        0x3f90, 0xffff0001, 0xff000000,
 298        0x9178, 0x0fff0fff, 0x00070000,
 299        0x9194, 0x0fff0fff, 0x00070000,
 300        0x9148, 0xffff0001, 0xff000000,
 301        0x9190, 0x0fff0fff, 0x00090008,
 302        0x91ac, 0x0fff0fff, 0x00090008,
 303        0x3f94, 0xffff0000, 0xff000000,
 304        0x914c, 0xffff0000, 0xff000000,
 305        0x929c, 0x00000fff, 0x00000001,
 306        0x55e4, 0xff607fff, 0xfc000100,
 307        0x8a18, 0xff000fff, 0x00000100,
 308        0x8b28, 0xff000fff, 0x00000100,
 309        0x9144, 0xfffc0fff, 0x00000100,
 310        0x6ed8, 0x00010101, 0x00010000,
 311        0x9830, 0xffffffff, 0x00000000,
 312        0x9834, 0xf00fffff, 0x00000400,
 313        0x9838, 0xfffffffe, 0x00000000,
 314        0xd0c0, 0xff000fff, 0x00000100,
 315        0xd02c, 0xbfffff1f, 0x08421000,
 316        0xd0b8, 0x73773777, 0x12010001,
 317        0x5bb0, 0x000000f0, 0x00000070,
 318        0x98f8, 0x73773777, 0x12010001,
 319        0x98fc, 0xffffffff, 0x00000010,
 320        0x9b7c, 0x00ff0000, 0x00fc0000,
 321        0x8030, 0x00001f0f, 0x0000100a,
 322        0x2f48, 0x73773777, 0x12010001,
 323        0x2408, 0x00030000, 0x000c007f,
 324        0x8a14, 0xf000003f, 0x00000007,
 325        0x8b24, 0x3fff3fff, 0x00ff0fff,
 326        0x8b10, 0x0000ff0f, 0x00000000,
 327        0x28a4c, 0x07ffffff, 0x06000000,
 328        0x4d8, 0x00000fff, 0x00000100,
 329        0xa008, 0xffffffff, 0x00010000,
 330        0x913c, 0xffff03ff, 0x01000100,
 331        0x8c00, 0x000000ff, 0x00000003,
 332        0x8c04, 0xf8ff00ff, 0x40600060,
 333        0x8cf0, 0x1fff1fff, 0x08e00410,
 334        0x28350, 0x00000f01, 0x00000000,
 335        0x9508, 0xf700071f, 0x00000002,
 336        0x960c, 0xffffffff, 0x54763210,
 337        0x20ef8, 0x01ff01ff, 0x00000002,
 338        0x20e98, 0xfffffbff, 0x00200000,
 339        0x2015c, 0xffffffff, 0x00000f40,
 340        0x88c4, 0x001f3ae3, 0x00000082,
 341        0x8978, 0x3fffffff, 0x04050140,
 342        0x88d4, 0x0000001f, 0x00000010,
 343        0x8974, 0xffffffff, 0x00000000
 344};
 345
 346static const u32 scrapper_golden_registers[] =
 347{
 348        0x690, 0x3fff3fff, 0x20c00033,
 349        0x918c, 0x0fff0fff, 0x00010006,
 350        0x918c, 0x0fff0fff, 0x00010006,
 351        0x91a8, 0x0fff0fff, 0x00010006,
 352        0x91a8, 0x0fff0fff, 0x00010006,
 353        0x9150, 0xffffdfff, 0x6e944040,
 354        0x9150, 0xffffdfff, 0x6e944040,
 355        0x917c, 0x0fff0fff, 0x00030002,
 356        0x917c, 0x0fff0fff, 0x00030002,
 357        0x9198, 0x0fff0fff, 0x00030002,
 358        0x9198, 0x0fff0fff, 0x00030002,
 359        0x915c, 0x0fff0fff, 0x00010000,
 360        0x915c, 0x0fff0fff, 0x00010000,
 361        0x3f90, 0xffff0001, 0xff000000,
 362        0x3f90, 0xffff0001, 0xff000000,
 363        0x9178, 0x0fff0fff, 0x00070000,
 364        0x9178, 0x0fff0fff, 0x00070000,
 365        0x9194, 0x0fff0fff, 0x00070000,
 366        0x9194, 0x0fff0fff, 0x00070000,
 367        0x9148, 0xffff0001, 0xff000000,
 368        0x9148, 0xffff0001, 0xff000000,
 369        0x9190, 0x0fff0fff, 0x00090008,
 370        0x9190, 0x0fff0fff, 0x00090008,
 371        0x91ac, 0x0fff0fff, 0x00090008,
 372        0x91ac, 0x0fff0fff, 0x00090008,
 373        0x3f94, 0xffff0000, 0xff000000,
 374        0x3f94, 0xffff0000, 0xff000000,
 375        0x914c, 0xffff0000, 0xff000000,
 376        0x914c, 0xffff0000, 0xff000000,
 377        0x929c, 0x00000fff, 0x00000001,
 378        0x929c, 0x00000fff, 0x00000001,
 379        0x55e4, 0xff607fff, 0xfc000100,
 380        0x8a18, 0xff000fff, 0x00000100,
 381        0x8a18, 0xff000fff, 0x00000100,
 382        0x8b28, 0xff000fff, 0x00000100,
 383        0x8b28, 0xff000fff, 0x00000100,
 384        0x9144, 0xfffc0fff, 0x00000100,
 385        0x9144, 0xfffc0fff, 0x00000100,
 386        0x6ed8, 0x00010101, 0x00010000,
 387        0x9830, 0xffffffff, 0x00000000,
 388        0x9830, 0xffffffff, 0x00000000,
 389        0x9834, 0xf00fffff, 0x00000400,
 390        0x9834, 0xf00fffff, 0x00000400,
 391        0x9838, 0xfffffffe, 0x00000000,
 392        0x9838, 0xfffffffe, 0x00000000,
 393        0xd0c0, 0xff000fff, 0x00000100,
 394        0xd02c, 0xbfffff1f, 0x08421000,
 395        0xd02c, 0xbfffff1f, 0x08421000,
 396        0xd0b8, 0x73773777, 0x12010001,
 397        0xd0b8, 0x73773777, 0x12010001,
 398        0x5bb0, 0x000000f0, 0x00000070,
 399        0x98f8, 0x73773777, 0x12010001,
 400        0x98f8, 0x73773777, 0x12010001,
 401        0x98fc, 0xffffffff, 0x00000010,
 402        0x98fc, 0xffffffff, 0x00000010,
 403        0x9b7c, 0x00ff0000, 0x00fc0000,
 404        0x9b7c, 0x00ff0000, 0x00fc0000,
 405        0x8030, 0x00001f0f, 0x0000100a,
 406        0x8030, 0x00001f0f, 0x0000100a,
 407        0x2f48, 0x73773777, 0x12010001,
 408        0x2f48, 0x73773777, 0x12010001,
 409        0x2408, 0x00030000, 0x000c007f,
 410        0x8a14, 0xf000003f, 0x00000007,
 411        0x8a14, 0xf000003f, 0x00000007,
 412        0x8b24, 0x3fff3fff, 0x00ff0fff,
 413        0x8b24, 0x3fff3fff, 0x00ff0fff,
 414        0x8b10, 0x0000ff0f, 0x00000000,
 415        0x8b10, 0x0000ff0f, 0x00000000,
 416        0x28a4c, 0x07ffffff, 0x06000000,
 417        0x28a4c, 0x07ffffff, 0x06000000,
 418        0x4d8, 0x00000fff, 0x00000100,
 419        0x4d8, 0x00000fff, 0x00000100,
 420        0xa008, 0xffffffff, 0x00010000,
 421        0xa008, 0xffffffff, 0x00010000,
 422        0x913c, 0xffff03ff, 0x01000100,
 423        0x913c, 0xffff03ff, 0x01000100,
 424        0x90e8, 0x001fffff, 0x010400c0,
 425        0x8c00, 0x000000ff, 0x00000003,
 426        0x8c00, 0x000000ff, 0x00000003,
 427        0x8c04, 0xf8ff00ff, 0x40600060,
 428        0x8c04, 0xf8ff00ff, 0x40600060,
 429        0x8c30, 0x0000000f, 0x00040005,
 430        0x8cf0, 0x1fff1fff, 0x08e00410,
 431        0x8cf0, 0x1fff1fff, 0x08e00410,
 432        0x900c, 0x00ffffff, 0x0017071f,
 433        0x28350, 0x00000f01, 0x00000000,
 434        0x28350, 0x00000f01, 0x00000000,
 435        0x9508, 0xf700071f, 0x00000002,
 436        0x9508, 0xf700071f, 0x00000002,
 437        0x9688, 0x00300000, 0x0017000f,
 438        0x960c, 0xffffffff, 0x54763210,
 439        0x960c, 0xffffffff, 0x54763210,
 440        0x20ef8, 0x01ff01ff, 0x00000002,
 441        0x20e98, 0xfffffbff, 0x00200000,
 442        0x2015c, 0xffffffff, 0x00000f40,
 443        0x88c4, 0x001f3ae3, 0x00000082,
 444        0x88c4, 0x001f3ae3, 0x00000082,
 445        0x8978, 0x3fffffff, 0x04050140,
 446        0x8978, 0x3fffffff, 0x04050140,
 447        0x88d4, 0x0000001f, 0x00000010,
 448        0x88d4, 0x0000001f, 0x00000010,
 449        0x8974, 0xffffffff, 0x00000000,
 450        0x8974, 0xffffffff, 0x00000000
 451};
 452
 453static void ni_init_golden_registers(struct radeon_device *rdev)
 454{
 455        switch (rdev->family) {
 456        case CHIP_CAYMAN:
 457                radeon_program_register_sequence(rdev,
 458                                                 cayman_golden_registers,
 459                                                 (const u32)ARRAY_SIZE(cayman_golden_registers));
 460                radeon_program_register_sequence(rdev,
 461                                                 cayman_golden_registers2,
 462                                                 (const u32)ARRAY_SIZE(cayman_golden_registers2));
 463                break;
 464        case CHIP_ARUBA:
 465                if ((rdev->pdev->device == 0x9900) ||
 466                    (rdev->pdev->device == 0x9901) ||
 467                    (rdev->pdev->device == 0x9903) ||
 468                    (rdev->pdev->device == 0x9904) ||
 469                    (rdev->pdev->device == 0x9905) ||
 470                    (rdev->pdev->device == 0x9906) ||
 471                    (rdev->pdev->device == 0x9907) ||
 472                    (rdev->pdev->device == 0x9908) ||
 473                    (rdev->pdev->device == 0x9909) ||
 474                    (rdev->pdev->device == 0x990A) ||
 475                    (rdev->pdev->device == 0x990B) ||
 476                    (rdev->pdev->device == 0x990C) ||
 477                    (rdev->pdev->device == 0x990D) ||
 478                    (rdev->pdev->device == 0x990E) ||
 479                    (rdev->pdev->device == 0x990F) ||
 480                    (rdev->pdev->device == 0x9910) ||
 481                    (rdev->pdev->device == 0x9913) ||
 482                    (rdev->pdev->device == 0x9917) ||
 483                    (rdev->pdev->device == 0x9918)) {
 484                        radeon_program_register_sequence(rdev,
 485                                                         dvst_golden_registers,
 486                                                         (const u32)ARRAY_SIZE(dvst_golden_registers));
 487                        radeon_program_register_sequence(rdev,
 488                                                         dvst_golden_registers2,
 489                                                         (const u32)ARRAY_SIZE(dvst_golden_registers2));
 490                } else {
 491                        radeon_program_register_sequence(rdev,
 492                                                         scrapper_golden_registers,
 493                                                         (const u32)ARRAY_SIZE(scrapper_golden_registers));
 494                        radeon_program_register_sequence(rdev,
 495                                                         dvst_golden_registers2,
 496                                                         (const u32)ARRAY_SIZE(dvst_golden_registers2));
 497                }
 498                break;
 499        default:
 500                break;
 501        }
 502}
 503
 504#define BTC_IO_MC_REGS_SIZE 29
 505
 506static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 507        {0x00000077, 0xff010100},
 508        {0x00000078, 0x00000000},
 509        {0x00000079, 0x00001434},
 510        {0x0000007a, 0xcc08ec08},
 511        {0x0000007b, 0x00040000},
 512        {0x0000007c, 0x000080c0},
 513        {0x0000007d, 0x09000000},
 514        {0x0000007e, 0x00210404},
 515        {0x00000081, 0x08a8e800},
 516        {0x00000082, 0x00030444},
 517        {0x00000083, 0x00000000},
 518        {0x00000085, 0x00000001},
 519        {0x00000086, 0x00000002},
 520        {0x00000087, 0x48490000},
 521        {0x00000088, 0x20244647},
 522        {0x00000089, 0x00000005},
 523        {0x0000008b, 0x66030000},
 524        {0x0000008c, 0x00006603},
 525        {0x0000008d, 0x00000100},
 526        {0x0000008f, 0x00001c0a},
 527        {0x00000090, 0xff000001},
 528        {0x00000094, 0x00101101},
 529        {0x00000095, 0x00000fff},
 530        {0x00000096, 0x00116fff},
 531        {0x00000097, 0x60010000},
 532        {0x00000098, 0x10010000},
 533        {0x00000099, 0x00006000},
 534        {0x0000009a, 0x00001000},
 535        {0x0000009f, 0x00946a00}
 536};
 537
 538static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 539        {0x00000077, 0xff010100},
 540        {0x00000078, 0x00000000},
 541        {0x00000079, 0x00001434},
 542        {0x0000007a, 0xcc08ec08},
 543        {0x0000007b, 0x00040000},
 544        {0x0000007c, 0x000080c0},
 545        {0x0000007d, 0x09000000},
 546        {0x0000007e, 0x00210404},
 547        {0x00000081, 0x08a8e800},
 548        {0x00000082, 0x00030444},
 549        {0x00000083, 0x00000000},
 550        {0x00000085, 0x00000001},
 551        {0x00000086, 0x00000002},
 552        {0x00000087, 0x48490000},
 553        {0x00000088, 0x20244647},
 554        {0x00000089, 0x00000005},
 555        {0x0000008b, 0x66030000},
 556        {0x0000008c, 0x00006603},
 557        {0x0000008d, 0x00000100},
 558        {0x0000008f, 0x00001c0a},
 559        {0x00000090, 0xff000001},
 560        {0x00000094, 0x00101101},
 561        {0x00000095, 0x00000fff},
 562        {0x00000096, 0x00116fff},
 563        {0x00000097, 0x60010000},
 564        {0x00000098, 0x10010000},
 565        {0x00000099, 0x00006000},
 566        {0x0000009a, 0x00001000},
 567        {0x0000009f, 0x00936a00}
 568};
 569
 570static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 571        {0x00000077, 0xff010100},
 572        {0x00000078, 0x00000000},
 573        {0x00000079, 0x00001434},
 574        {0x0000007a, 0xcc08ec08},
 575        {0x0000007b, 0x00040000},
 576        {0x0000007c, 0x000080c0},
 577        {0x0000007d, 0x09000000},
 578        {0x0000007e, 0x00210404},
 579        {0x00000081, 0x08a8e800},
 580        {0x00000082, 0x00030444},
 581        {0x00000083, 0x00000000},
 582        {0x00000085, 0x00000001},
 583        {0x00000086, 0x00000002},
 584        {0x00000087, 0x48490000},
 585        {0x00000088, 0x20244647},
 586        {0x00000089, 0x00000005},
 587        {0x0000008b, 0x66030000},
 588        {0x0000008c, 0x00006603},
 589        {0x0000008d, 0x00000100},
 590        {0x0000008f, 0x00001c0a},
 591        {0x00000090, 0xff000001},
 592        {0x00000094, 0x00101101},
 593        {0x00000095, 0x00000fff},
 594        {0x00000096, 0x00116fff},
 595        {0x00000097, 0x60010000},
 596        {0x00000098, 0x10010000},
 597        {0x00000099, 0x00006000},
 598        {0x0000009a, 0x00001000},
 599        {0x0000009f, 0x00916a00}
 600};
 601
 602static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
 603        {0x00000077, 0xff010100},
 604        {0x00000078, 0x00000000},
 605        {0x00000079, 0x00001434},
 606        {0x0000007a, 0xcc08ec08},
 607        {0x0000007b, 0x00040000},
 608        {0x0000007c, 0x000080c0},
 609        {0x0000007d, 0x09000000},
 610        {0x0000007e, 0x00210404},
 611        {0x00000081, 0x08a8e800},
 612        {0x00000082, 0x00030444},
 613        {0x00000083, 0x00000000},
 614        {0x00000085, 0x00000001},
 615        {0x00000086, 0x00000002},
 616        {0x00000087, 0x48490000},
 617        {0x00000088, 0x20244647},
 618        {0x00000089, 0x00000005},
 619        {0x0000008b, 0x66030000},
 620        {0x0000008c, 0x00006603},
 621        {0x0000008d, 0x00000100},
 622        {0x0000008f, 0x00001c0a},
 623        {0x00000090, 0xff000001},
 624        {0x00000094, 0x00101101},
 625        {0x00000095, 0x00000fff},
 626        {0x00000096, 0x00116fff},
 627        {0x00000097, 0x60010000},
 628        {0x00000098, 0x10010000},
 629        {0x00000099, 0x00006000},
 630        {0x0000009a, 0x00001000},
 631        {0x0000009f, 0x00976b00}
 632};
 633
 634int ni_mc_load_microcode(struct radeon_device *rdev)
 635{
 636        const __be32 *fw_data;
 637        u32 mem_type, running, blackout = 0;
 638        u32 *io_mc_regs;
 639        int i, ucode_size, regs_size;
 640
 641        if (!rdev->mc_fw)
 642                return -EINVAL;
 643
 644        switch (rdev->family) {
 645        case CHIP_BARTS:
 646                io_mc_regs = (u32 *)&barts_io_mc_regs;
 647                ucode_size = BTC_MC_UCODE_SIZE;
 648                regs_size = BTC_IO_MC_REGS_SIZE;
 649                break;
 650        case CHIP_TURKS:
 651                io_mc_regs = (u32 *)&turks_io_mc_regs;
 652                ucode_size = BTC_MC_UCODE_SIZE;
 653                regs_size = BTC_IO_MC_REGS_SIZE;
 654                break;
 655        case CHIP_CAICOS:
 656        default:
 657                io_mc_regs = (u32 *)&caicos_io_mc_regs;
 658                ucode_size = BTC_MC_UCODE_SIZE;
 659                regs_size = BTC_IO_MC_REGS_SIZE;
 660                break;
 661        case CHIP_CAYMAN:
 662                io_mc_regs = (u32 *)&cayman_io_mc_regs;
 663                ucode_size = CAYMAN_MC_UCODE_SIZE;
 664                regs_size = BTC_IO_MC_REGS_SIZE;
 665                break;
 666        }
 667
 668        mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
 669        running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
 670
 671        if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
 672                if (running) {
 673                        blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
 674                        WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
 675                }
 676
 677                /* reset the engine and set to writable */
 678                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 679                WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
 680
 681                /* load mc io regs */
 682                for (i = 0; i < regs_size; i++) {
 683                        WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
 684                        WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
 685                }
 686                /* load the MC ucode */
 687                fw_data = (const __be32 *)rdev->mc_fw->data;
 688                for (i = 0; i < ucode_size; i++)
 689                        WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
 690
 691                /* put the engine back into the active state */
 692                WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
 693                WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
 694                WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
 695
 696                /* wait for training to complete */
 697                for (i = 0; i < rdev->usec_timeout; i++) {
 698                        if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
 699                                break;
 700                        udelay(1);
 701                }
 702
 703                if (running)
 704                        WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
 705        }
 706
 707        return 0;
 708}
 709
 710int ni_init_microcode(struct radeon_device *rdev)
 711{
 712        const char *chip_name;
 713        const char *rlc_chip_name;
 714        size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
 715        size_t smc_req_size = 0;
 716        char fw_name[30];
 717        int err;
 718
 719        DRM_DEBUG("\n");
 720
 721        switch (rdev->family) {
 722        case CHIP_BARTS:
 723                chip_name = "BARTS";
 724                rlc_chip_name = "BTC";
 725                pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 726                me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 727                rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 728                mc_req_size = BTC_MC_UCODE_SIZE * 4;
 729                smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
 730                break;
 731        case CHIP_TURKS:
 732                chip_name = "TURKS";
 733                rlc_chip_name = "BTC";
 734                pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 735                me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 736                rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 737                mc_req_size = BTC_MC_UCODE_SIZE * 4;
 738                smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
 739                break;
 740        case CHIP_CAICOS:
 741                chip_name = "CAICOS";
 742                rlc_chip_name = "BTC";
 743                pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
 744                me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
 745                rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
 746                mc_req_size = BTC_MC_UCODE_SIZE * 4;
 747                smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
 748                break;
 749        case CHIP_CAYMAN:
 750                chip_name = "CAYMAN";
 751                rlc_chip_name = "CAYMAN";
 752                pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 753                me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 754                rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
 755                mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
 756                smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
 757                break;
 758        case CHIP_ARUBA:
 759                chip_name = "ARUBA";
 760                rlc_chip_name = "ARUBA";
 761                /* pfp/me same size as CAYMAN */
 762                pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
 763                me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
 764                rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
 765                mc_req_size = 0;
 766                break;
 767        default: BUG();
 768        }
 769
 770        DRM_INFO("Loading %s Microcode\n", chip_name);
 771
 772        snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
 773        err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
 774        if (err)
 775                goto out;
 776        if (rdev->pfp_fw->size != pfp_req_size) {
 777                pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 778                       rdev->pfp_fw->size, fw_name);
 779                err = -EINVAL;
 780                goto out;
 781        }
 782
 783        snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
 784        err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
 785        if (err)
 786                goto out;
 787        if (rdev->me_fw->size != me_req_size) {
 788                pr_err("ni_cp: Bogus length %zu in firmware \"%s\"\n",
 789                       rdev->me_fw->size, fw_name);
 790                err = -EINVAL;
 791        }
 792
 793        snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
 794        err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
 795        if (err)
 796                goto out;
 797        if (rdev->rlc_fw->size != rlc_req_size) {
 798                pr_err("ni_rlc: Bogus length %zu in firmware \"%s\"\n",
 799                       rdev->rlc_fw->size, fw_name);
 800                err = -EINVAL;
 801        }
 802
 803        /* no MC ucode on TN */
 804        if (!(rdev->flags & RADEON_IS_IGP)) {
 805                snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
 806                err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
 807                if (err)
 808                        goto out;
 809                if (rdev->mc_fw->size != mc_req_size) {
 810                        pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 811                               rdev->mc_fw->size, fw_name);
 812                        err = -EINVAL;
 813                }
 814        }
 815
 816        if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
 817                snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
 818                err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
 819                if (err) {
 820                        pr_err("smc: error loading firmware \"%s\"\n", fw_name);
 821                        release_firmware(rdev->smc_fw);
 822                        rdev->smc_fw = NULL;
 823                        err = 0;
 824                } else if (rdev->smc_fw->size != smc_req_size) {
 825                        pr_err("ni_mc: Bogus length %zu in firmware \"%s\"\n",
 826                               rdev->mc_fw->size, fw_name);
 827                        err = -EINVAL;
 828                }
 829        }
 830
 831out:
 832        if (err) {
 833                if (err != -EINVAL)
 834                        pr_err("ni_cp: Failed to load firmware \"%s\"\n",
 835                               fw_name);
 836                release_firmware(rdev->pfp_fw);
 837                rdev->pfp_fw = NULL;
 838                release_firmware(rdev->me_fw);
 839                rdev->me_fw = NULL;
 840                release_firmware(rdev->rlc_fw);
 841                rdev->rlc_fw = NULL;
 842                release_firmware(rdev->mc_fw);
 843                rdev->mc_fw = NULL;
 844        }
 845        return err;
 846}
 847
 848/**
 849 * cayman_get_allowed_info_register - fetch the register for the info ioctl
 850 *
 851 * @rdev: radeon_device pointer
 852 * @reg: register offset in bytes
 853 * @val: register value
 854 *
 855 * Returns 0 for success or -EINVAL for an invalid register
 856 *
 857 */
 858int cayman_get_allowed_info_register(struct radeon_device *rdev,
 859                                     u32 reg, u32 *val)
 860{
 861        switch (reg) {
 862        case GRBM_STATUS:
 863        case GRBM_STATUS_SE0:
 864        case GRBM_STATUS_SE1:
 865        case SRBM_STATUS:
 866        case SRBM_STATUS2:
 867        case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
 868        case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
 869        case UVD_STATUS:
 870                *val = RREG32(reg);
 871                return 0;
 872        default:
 873                return -EINVAL;
 874        }
 875}
 876
 877int tn_get_temp(struct radeon_device *rdev)
 878{
 879        u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
 880        int actual_temp = (temp / 8) - 49;
 881
 882        return actual_temp * 1000;
 883}
 884
 885/*
 886 * Core functions
 887 */
 888static void cayman_gpu_init(struct radeon_device *rdev)
 889{
 890        u32 gb_addr_config = 0;
 891        u32 mc_shared_chmap, mc_arb_ramcfg;
 892        u32 cgts_tcc_disable;
 893        u32 sx_debug_1;
 894        u32 smx_dc_ctl0;
 895        u32 cgts_sm_ctrl_reg;
 896        u32 hdp_host_path_cntl;
 897        u32 tmp;
 898        u32 disabled_rb_mask;
 899        int i, j;
 900
 901        switch (rdev->family) {
 902        case CHIP_CAYMAN:
 903                rdev->config.cayman.max_shader_engines = 2;
 904                rdev->config.cayman.max_pipes_per_simd = 4;
 905                rdev->config.cayman.max_tile_pipes = 8;
 906                rdev->config.cayman.max_simds_per_se = 12;
 907                rdev->config.cayman.max_backends_per_se = 4;
 908                rdev->config.cayman.max_texture_channel_caches = 8;
 909                rdev->config.cayman.max_gprs = 256;
 910                rdev->config.cayman.max_threads = 256;
 911                rdev->config.cayman.max_gs_threads = 32;
 912                rdev->config.cayman.max_stack_entries = 512;
 913                rdev->config.cayman.sx_num_of_sets = 8;
 914                rdev->config.cayman.sx_max_export_size = 256;
 915                rdev->config.cayman.sx_max_export_pos_size = 64;
 916                rdev->config.cayman.sx_max_export_smx_size = 192;
 917                rdev->config.cayman.max_hw_contexts = 8;
 918                rdev->config.cayman.sq_num_cf_insts = 2;
 919
 920                rdev->config.cayman.sc_prim_fifo_size = 0x100;
 921                rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 922                rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 923                gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
 924                break;
 925        case CHIP_ARUBA:
 926        default:
 927                rdev->config.cayman.max_shader_engines = 1;
 928                rdev->config.cayman.max_pipes_per_simd = 4;
 929                rdev->config.cayman.max_tile_pipes = 2;
 930                if ((rdev->pdev->device == 0x9900) ||
 931                    (rdev->pdev->device == 0x9901) ||
 932                    (rdev->pdev->device == 0x9905) ||
 933                    (rdev->pdev->device == 0x9906) ||
 934                    (rdev->pdev->device == 0x9907) ||
 935                    (rdev->pdev->device == 0x9908) ||
 936                    (rdev->pdev->device == 0x9909) ||
 937                    (rdev->pdev->device == 0x990B) ||
 938                    (rdev->pdev->device == 0x990C) ||
 939                    (rdev->pdev->device == 0x990F) ||
 940                    (rdev->pdev->device == 0x9910) ||
 941                    (rdev->pdev->device == 0x9917) ||
 942                    (rdev->pdev->device == 0x9999) ||
 943                    (rdev->pdev->device == 0x999C)) {
 944                        rdev->config.cayman.max_simds_per_se = 6;
 945                        rdev->config.cayman.max_backends_per_se = 2;
 946                        rdev->config.cayman.max_hw_contexts = 8;
 947                        rdev->config.cayman.sx_max_export_size = 256;
 948                        rdev->config.cayman.sx_max_export_pos_size = 64;
 949                        rdev->config.cayman.sx_max_export_smx_size = 192;
 950                } else if ((rdev->pdev->device == 0x9903) ||
 951                           (rdev->pdev->device == 0x9904) ||
 952                           (rdev->pdev->device == 0x990A) ||
 953                           (rdev->pdev->device == 0x990D) ||
 954                           (rdev->pdev->device == 0x990E) ||
 955                           (rdev->pdev->device == 0x9913) ||
 956                           (rdev->pdev->device == 0x9918) ||
 957                           (rdev->pdev->device == 0x999D)) {
 958                        rdev->config.cayman.max_simds_per_se = 4;
 959                        rdev->config.cayman.max_backends_per_se = 2;
 960                        rdev->config.cayman.max_hw_contexts = 8;
 961                        rdev->config.cayman.sx_max_export_size = 256;
 962                        rdev->config.cayman.sx_max_export_pos_size = 64;
 963                        rdev->config.cayman.sx_max_export_smx_size = 192;
 964                } else if ((rdev->pdev->device == 0x9919) ||
 965                           (rdev->pdev->device == 0x9990) ||
 966                           (rdev->pdev->device == 0x9991) ||
 967                           (rdev->pdev->device == 0x9994) ||
 968                           (rdev->pdev->device == 0x9995) ||
 969                           (rdev->pdev->device == 0x9996) ||
 970                           (rdev->pdev->device == 0x999A) ||
 971                           (rdev->pdev->device == 0x99A0)) {
 972                        rdev->config.cayman.max_simds_per_se = 3;
 973                        rdev->config.cayman.max_backends_per_se = 1;
 974                        rdev->config.cayman.max_hw_contexts = 4;
 975                        rdev->config.cayman.sx_max_export_size = 128;
 976                        rdev->config.cayman.sx_max_export_pos_size = 32;
 977                        rdev->config.cayman.sx_max_export_smx_size = 96;
 978                } else {
 979                        rdev->config.cayman.max_simds_per_se = 2;
 980                        rdev->config.cayman.max_backends_per_se = 1;
 981                        rdev->config.cayman.max_hw_contexts = 4;
 982                        rdev->config.cayman.sx_max_export_size = 128;
 983                        rdev->config.cayman.sx_max_export_pos_size = 32;
 984                        rdev->config.cayman.sx_max_export_smx_size = 96;
 985                }
 986                rdev->config.cayman.max_texture_channel_caches = 2;
 987                rdev->config.cayman.max_gprs = 256;
 988                rdev->config.cayman.max_threads = 256;
 989                rdev->config.cayman.max_gs_threads = 32;
 990                rdev->config.cayman.max_stack_entries = 512;
 991                rdev->config.cayman.sx_num_of_sets = 8;
 992                rdev->config.cayman.sq_num_cf_insts = 2;
 993
 994                rdev->config.cayman.sc_prim_fifo_size = 0x40;
 995                rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
 996                rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
 997                gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
 998                break;
 999        }
1000
1001        /* Initialize HDP */
1002        for (i = 0, j = 0; i < 32; i++, j += 0x18) {
1003                WREG32((0x2c14 + j), 0x00000000);
1004                WREG32((0x2c18 + j), 0x00000000);
1005                WREG32((0x2c1c + j), 0x00000000);
1006                WREG32((0x2c20 + j), 0x00000000);
1007                WREG32((0x2c24 + j), 0x00000000);
1008        }
1009
1010        WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
1011        WREG32(SRBM_INT_CNTL, 0x1);
1012        WREG32(SRBM_INT_ACK, 0x1);
1013
1014        evergreen_fix_pci_max_read_req_size(rdev);
1015
1016        mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
1017        mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
1018
1019        tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
1020        rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1021        if (rdev->config.cayman.mem_row_size_in_kb > 4)
1022                rdev->config.cayman.mem_row_size_in_kb = 4;
1023        /* XXX use MC settings? */
1024        rdev->config.cayman.shader_engine_tile_size = 32;
1025        rdev->config.cayman.num_gpus = 1;
1026        rdev->config.cayman.multi_gpu_tile_size = 64;
1027
1028        tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
1029        rdev->config.cayman.num_tile_pipes = (1 << tmp);
1030        tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
1031        rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
1032        tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
1033        rdev->config.cayman.num_shader_engines = tmp + 1;
1034        tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
1035        rdev->config.cayman.num_gpus = tmp + 1;
1036        tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
1037        rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
1038        tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
1039        rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
1040
1041
1042        /* setup tiling info dword.  gb_addr_config is not adequate since it does
1043         * not have bank info, so create a custom tiling dword.
1044         * bits 3:0   num_pipes
1045         * bits 7:4   num_banks
1046         * bits 11:8  group_size
1047         * bits 15:12 row_size
1048         */
1049        rdev->config.cayman.tile_config = 0;
1050        switch (rdev->config.cayman.num_tile_pipes) {
1051        case 1:
1052        default:
1053                rdev->config.cayman.tile_config |= (0 << 0);
1054                break;
1055        case 2:
1056                rdev->config.cayman.tile_config |= (1 << 0);
1057                break;
1058        case 4:
1059                rdev->config.cayman.tile_config |= (2 << 0);
1060                break;
1061        case 8:
1062                rdev->config.cayman.tile_config |= (3 << 0);
1063                break;
1064        }
1065
1066        /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1067        if (rdev->flags & RADEON_IS_IGP)
1068                rdev->config.cayman.tile_config |= 1 << 4;
1069        else {
1070                switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1071                case 0: /* four banks */
1072                        rdev->config.cayman.tile_config |= 0 << 4;
1073                        break;
1074                case 1: /* eight banks */
1075                        rdev->config.cayman.tile_config |= 1 << 4;
1076                        break;
1077                case 2: /* sixteen banks */
1078                default:
1079                        rdev->config.cayman.tile_config |= 2 << 4;
1080                        break;
1081                }
1082        }
1083        rdev->config.cayman.tile_config |=
1084                ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1085        rdev->config.cayman.tile_config |=
1086                ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1087
1088        tmp = 0;
1089        for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1090                u32 rb_disable_bitmap;
1091
1092                WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1093                WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1094                rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1095                tmp <<= 4;
1096                tmp |= rb_disable_bitmap;
1097        }
1098        /* enabled rb are just the one not disabled :) */
1099        disabled_rb_mask = tmp;
1100        tmp = 0;
1101        for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1102                tmp |= (1 << i);
1103        /* if all the backends are disabled, fix it up here */
1104        if ((disabled_rb_mask & tmp) == tmp) {
1105                for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1106                        disabled_rb_mask &= ~(1 << i);
1107        }
1108
1109        for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1110                u32 simd_disable_bitmap;
1111
1112                WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1113                WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1114                simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1115                simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1116                tmp <<= 16;
1117                tmp |= simd_disable_bitmap;
1118        }
1119        rdev->config.cayman.active_simds = hweight32(~tmp);
1120
1121        WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1122        WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1123
1124        WREG32(GB_ADDR_CONFIG, gb_addr_config);
1125        WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1126        if (ASIC_IS_DCE6(rdev))
1127                WREG32(DMIF_ADDR_CALC, gb_addr_config);
1128        WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1129        WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1130        WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1131        WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1132        WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1133        WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1134
1135        if ((rdev->config.cayman.max_backends_per_se == 1) &&
1136            (rdev->flags & RADEON_IS_IGP)) {
1137                if ((disabled_rb_mask & 3) == 2) {
1138                        /* RB1 disabled, RB0 enabled */
1139                        tmp = 0x00000000;
1140                } else {
1141                        /* RB0 disabled, RB1 enabled */
1142                        tmp = 0x11111111;
1143                }
1144        } else {
1145                tmp = gb_addr_config & NUM_PIPES_MASK;
1146                tmp = r6xx_remap_render_backend(rdev, tmp,
1147                                                rdev->config.cayman.max_backends_per_se *
1148                                                rdev->config.cayman.max_shader_engines,
1149                                                CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1150        }
1151        WREG32(GB_BACKEND_MAP, tmp);
1152
1153        cgts_tcc_disable = 0xffff0000;
1154        for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1155                cgts_tcc_disable &= ~(1 << (16 + i));
1156        WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1157        WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1158        WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1159        WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1160
1161        /* reprogram the shader complex */
1162        cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1163        for (i = 0; i < 16; i++)
1164                WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1165        WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1166
1167        /* set HW defaults for 3D engine */
1168        WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1169
1170        sx_debug_1 = RREG32(SX_DEBUG_1);
1171        sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1172        WREG32(SX_DEBUG_1, sx_debug_1);
1173
1174        smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1175        smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1176        smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1177        WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1178
1179        WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1180
1181        /* need to be explicitly zero-ed */
1182        WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1183        WREG32(SQ_LSTMP_RING_BASE, 0);
1184        WREG32(SQ_HSTMP_RING_BASE, 0);
1185        WREG32(SQ_ESTMP_RING_BASE, 0);
1186        WREG32(SQ_GSTMP_RING_BASE, 0);
1187        WREG32(SQ_VSTMP_RING_BASE, 0);
1188        WREG32(SQ_PSTMP_RING_BASE, 0);
1189
1190        WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1191
1192        WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1193                                        POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1194                                        SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1195
1196        WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1197                                 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1198                                 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1199
1200
1201        WREG32(VGT_NUM_INSTANCES, 1);
1202
1203        WREG32(CP_PERFMON_CNTL, 0);
1204
1205        WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1206                                  FETCH_FIFO_HIWATER(0x4) |
1207                                  DONE_FIFO_HIWATER(0xe0) |
1208                                  ALU_UPDATE_FIFO_HIWATER(0x8)));
1209
1210        WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1211        WREG32(SQ_CONFIG, (VC_ENABLE |
1212                           EXPORT_SRC_C |
1213                           GFX_PRIO(0) |
1214                           CS1_PRIO(0) |
1215                           CS2_PRIO(1)));
1216        WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1217
1218        WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1219                                          FORCE_EOV_MAX_REZ_CNT(255)));
1220
1221        WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1222               AUTO_INVLD_EN(ES_AND_GS_AUTO));
1223
1224        WREG32(VGT_GS_VERTEX_REUSE, 16);
1225        WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1226
1227        WREG32(CB_PERF_CTR0_SEL_0, 0);
1228        WREG32(CB_PERF_CTR0_SEL_1, 0);
1229        WREG32(CB_PERF_CTR1_SEL_0, 0);
1230        WREG32(CB_PERF_CTR1_SEL_1, 0);
1231        WREG32(CB_PERF_CTR2_SEL_0, 0);
1232        WREG32(CB_PERF_CTR2_SEL_1, 0);
1233        WREG32(CB_PERF_CTR3_SEL_0, 0);
1234        WREG32(CB_PERF_CTR3_SEL_1, 0);
1235
1236        tmp = RREG32(HDP_MISC_CNTL);
1237        tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1238        WREG32(HDP_MISC_CNTL, tmp);
1239
1240        hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1241        WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1242
1243        WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1244
1245        udelay(50);
1246
1247        /* set clockgating golden values on TN */
1248        if (rdev->family == CHIP_ARUBA) {
1249                tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1250                tmp &= ~0x00380000;
1251                WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1252                tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1253                tmp &= ~0x0e000000;
1254                WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1255        }
1256}
1257
1258/*
1259 * GART
1260 */
1261void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1262{
1263        /* flush hdp cache */
1264        WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1265
1266        /* bits 0-7 are the VM contexts0-7 */
1267        WREG32(VM_INVALIDATE_REQUEST, 1);
1268}
1269
1270static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1271{
1272        int i, r;
1273
1274        if (rdev->gart.robj == NULL) {
1275                dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1276                return -EINVAL;
1277        }
1278        r = radeon_gart_table_vram_pin(rdev);
1279        if (r)
1280                return r;
1281        /* Setup TLB control */
1282        WREG32(MC_VM_MX_L1_TLB_CNTL,
1283               (0xA << 7) |
1284               ENABLE_L1_TLB |
1285               ENABLE_L1_FRAGMENT_PROCESSING |
1286               SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1287               ENABLE_ADVANCED_DRIVER_MODEL |
1288               SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1289        /* Setup L2 cache */
1290        WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1291               ENABLE_L2_FRAGMENT_PROCESSING |
1292               ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1293               ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1294               EFFECTIVE_L2_QUEUE_SIZE(7) |
1295               CONTEXT1_IDENTITY_ACCESS_MODE(1));
1296        WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1297        WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1298               BANK_SELECT(6) |
1299               L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1300        /* setup context0 */
1301        WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1302        WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1303        WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1304        WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1305                        (u32)(rdev->dummy_page.addr >> 12));
1306        WREG32(VM_CONTEXT0_CNTL2, 0);
1307        WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1308                                RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1309
1310        WREG32(0x15D4, 0);
1311        WREG32(0x15D8, 0);
1312        WREG32(0x15DC, 0);
1313
1314        /* empty context1-7 */
1315        /* Assign the pt base to something valid for now; the pts used for
1316         * the VMs are determined by the application and setup and assigned
1317         * on the fly in the vm part of radeon_gart.c
1318         */
1319        for (i = 1; i < 8; i++) {
1320                WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1321                WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2),
1322                        rdev->vm_manager.max_pfn - 1);
1323                WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1324                       rdev->vm_manager.saved_table_addr[i]);
1325        }
1326
1327        /* enable context1-7 */
1328        WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1329               (u32)(rdev->dummy_page.addr >> 12));
1330        WREG32(VM_CONTEXT1_CNTL2, 4);
1331        WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1332                                PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1333                                RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1334                                RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1335                                DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1336                                DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1337                                PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1338                                PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1339                                VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1340                                VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1341                                READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1342                                READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1343                                WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1344                                WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1345
1346        cayman_pcie_gart_tlb_flush(rdev);
1347        DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1348                 (unsigned)(rdev->mc.gtt_size >> 20),
1349                 (unsigned long long)rdev->gart.table_addr);
1350        rdev->gart.ready = true;
1351        return 0;
1352}
1353
1354static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1355{
1356        unsigned i;
1357
1358        for (i = 1; i < 8; ++i) {
1359                rdev->vm_manager.saved_table_addr[i] = RREG32(
1360                        VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1361        }
1362
1363        /* Disable all tables */
1364        WREG32(VM_CONTEXT0_CNTL, 0);
1365        WREG32(VM_CONTEXT1_CNTL, 0);
1366        /* Setup TLB control */
1367        WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1368               SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1369               SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1370        /* Setup L2 cache */
1371        WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1372               ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1373               EFFECTIVE_L2_QUEUE_SIZE(7) |
1374               CONTEXT1_IDENTITY_ACCESS_MODE(1));
1375        WREG32(VM_L2_CNTL2, 0);
1376        WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1377               L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1378        radeon_gart_table_vram_unpin(rdev);
1379}
1380
1381static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1382{
1383        cayman_pcie_gart_disable(rdev);
1384        radeon_gart_table_vram_free(rdev);
1385        radeon_gart_fini(rdev);
1386}
1387
1388void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1389                              int ring, u32 cp_int_cntl)
1390{
1391        WREG32(SRBM_GFX_CNTL, RINGID(ring));
1392        WREG32(CP_INT_CNTL, cp_int_cntl);
1393}
1394
1395/*
1396 * CP.
1397 */
1398void cayman_fence_ring_emit(struct radeon_device *rdev,
1399                            struct radeon_fence *fence)
1400{
1401        struct radeon_ring *ring = &rdev->ring[fence->ring];
1402        u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1403        u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1404                PACKET3_SH_ACTION_ENA;
1405
1406        /* flush read cache over gart for this vmid */
1407        radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1408        radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1409        radeon_ring_write(ring, 0xFFFFFFFF);
1410        radeon_ring_write(ring, 0);
1411        radeon_ring_write(ring, 10); /* poll interval */
1412        /* EVENT_WRITE_EOP - flush caches, send int */
1413        radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1414        radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1415        radeon_ring_write(ring, lower_32_bits(addr));
1416        radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1417        radeon_ring_write(ring, fence->seq);
1418        radeon_ring_write(ring, 0);
1419}
1420
1421void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1422{
1423        struct radeon_ring *ring = &rdev->ring[ib->ring];
1424        unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
1425        u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1426                PACKET3_SH_ACTION_ENA;
1427
1428        /* set to DX10/11 mode */
1429        radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1430        radeon_ring_write(ring, 1);
1431
1432        if (ring->rptr_save_reg) {
1433                uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1434                radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1435                radeon_ring_write(ring, ((ring->rptr_save_reg - 
1436                                          PACKET3_SET_CONFIG_REG_START) >> 2));
1437                radeon_ring_write(ring, next_rptr);
1438        }
1439
1440        radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1441        radeon_ring_write(ring,
1442#ifdef __BIG_ENDIAN
1443                          (2 << 0) |
1444#endif
1445                          (ib->gpu_addr & 0xFFFFFFFC));
1446        radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1447        radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
1448
1449        /* flush read cache over gart for this vmid */
1450        radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1451        radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1452        radeon_ring_write(ring, 0xFFFFFFFF);
1453        radeon_ring_write(ring, 0);
1454        radeon_ring_write(ring, (vm_id << 24) | 10); /* poll interval */
1455}
1456
1457static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1458{
1459        if (enable)
1460                WREG32(CP_ME_CNTL, 0);
1461        else {
1462                if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1463                        radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1464                WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1465                WREG32(SCRATCH_UMSK, 0);
1466                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1467        }
1468}
1469
1470u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1471                        struct radeon_ring *ring)
1472{
1473        u32 rptr;
1474
1475        if (rdev->wb.enabled)
1476                rptr = rdev->wb.wb[ring->rptr_offs/4];
1477        else {
1478                if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1479                        rptr = RREG32(CP_RB0_RPTR);
1480                else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1481                        rptr = RREG32(CP_RB1_RPTR);
1482                else
1483                        rptr = RREG32(CP_RB2_RPTR);
1484        }
1485
1486        return rptr;
1487}
1488
1489u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1490                        struct radeon_ring *ring)
1491{
1492        u32 wptr;
1493
1494        if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1495                wptr = RREG32(CP_RB0_WPTR);
1496        else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1497                wptr = RREG32(CP_RB1_WPTR);
1498        else
1499                wptr = RREG32(CP_RB2_WPTR);
1500
1501        return wptr;
1502}
1503
1504void cayman_gfx_set_wptr(struct radeon_device *rdev,
1505                         struct radeon_ring *ring)
1506{
1507        if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1508                WREG32(CP_RB0_WPTR, ring->wptr);
1509                (void)RREG32(CP_RB0_WPTR);
1510        } else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1511                WREG32(CP_RB1_WPTR, ring->wptr);
1512                (void)RREG32(CP_RB1_WPTR);
1513        } else {
1514                WREG32(CP_RB2_WPTR, ring->wptr);
1515                (void)RREG32(CP_RB2_WPTR);
1516        }
1517}
1518
1519static int cayman_cp_load_microcode(struct radeon_device *rdev)
1520{
1521        const __be32 *fw_data;
1522        int i;
1523
1524        if (!rdev->me_fw || !rdev->pfp_fw)
1525                return -EINVAL;
1526
1527        cayman_cp_enable(rdev, false);
1528
1529        fw_data = (const __be32 *)rdev->pfp_fw->data;
1530        WREG32(CP_PFP_UCODE_ADDR, 0);
1531        for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1532                WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1533        WREG32(CP_PFP_UCODE_ADDR, 0);
1534
1535        fw_data = (const __be32 *)rdev->me_fw->data;
1536        WREG32(CP_ME_RAM_WADDR, 0);
1537        for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1538                WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1539
1540        WREG32(CP_PFP_UCODE_ADDR, 0);
1541        WREG32(CP_ME_RAM_WADDR, 0);
1542        WREG32(CP_ME_RAM_RADDR, 0);
1543        return 0;
1544}
1545
1546static int cayman_cp_start(struct radeon_device *rdev)
1547{
1548        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1549        int r, i;
1550
1551        r = radeon_ring_lock(rdev, ring, 7);
1552        if (r) {
1553                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1554                return r;
1555        }
1556        radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1557        radeon_ring_write(ring, 0x1);
1558        radeon_ring_write(ring, 0x0);
1559        radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1560        radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1561        radeon_ring_write(ring, 0);
1562        radeon_ring_write(ring, 0);
1563        radeon_ring_unlock_commit(rdev, ring, false);
1564
1565        cayman_cp_enable(rdev, true);
1566
1567        r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1568        if (r) {
1569                DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1570                return r;
1571        }
1572
1573        /* setup clear context state */
1574        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1575        radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1576
1577        for (i = 0; i < cayman_default_size; i++)
1578                radeon_ring_write(ring, cayman_default_state[i]);
1579
1580        radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1581        radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1582
1583        /* set clear context state */
1584        radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1585        radeon_ring_write(ring, 0);
1586
1587        /* SQ_VTX_BASE_VTX_LOC */
1588        radeon_ring_write(ring, 0xc0026f00);
1589        radeon_ring_write(ring, 0x00000000);
1590        radeon_ring_write(ring, 0x00000000);
1591        radeon_ring_write(ring, 0x00000000);
1592
1593        /* Clear consts */
1594        radeon_ring_write(ring, 0xc0036f00);
1595        radeon_ring_write(ring, 0x00000bc4);
1596        radeon_ring_write(ring, 0xffffffff);
1597        radeon_ring_write(ring, 0xffffffff);
1598        radeon_ring_write(ring, 0xffffffff);
1599
1600        radeon_ring_write(ring, 0xc0026900);
1601        radeon_ring_write(ring, 0x00000316);
1602        radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1603        radeon_ring_write(ring, 0x00000010); /*  */
1604
1605        radeon_ring_unlock_commit(rdev, ring, false);
1606
1607        /* XXX init other rings */
1608
1609        return 0;
1610}
1611
1612static void cayman_cp_fini(struct radeon_device *rdev)
1613{
1614        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1615        cayman_cp_enable(rdev, false);
1616        radeon_ring_fini(rdev, ring);
1617        radeon_scratch_free(rdev, ring->rptr_save_reg);
1618}
1619
1620static int cayman_cp_resume(struct radeon_device *rdev)
1621{
1622        static const int ridx[] = {
1623                RADEON_RING_TYPE_GFX_INDEX,
1624                CAYMAN_RING_TYPE_CP1_INDEX,
1625                CAYMAN_RING_TYPE_CP2_INDEX
1626        };
1627        static const unsigned cp_rb_cntl[] = {
1628                CP_RB0_CNTL,
1629                CP_RB1_CNTL,
1630                CP_RB2_CNTL,
1631        };
1632        static const unsigned cp_rb_rptr_addr[] = {
1633                CP_RB0_RPTR_ADDR,
1634                CP_RB1_RPTR_ADDR,
1635                CP_RB2_RPTR_ADDR
1636        };
1637        static const unsigned cp_rb_rptr_addr_hi[] = {
1638                CP_RB0_RPTR_ADDR_HI,
1639                CP_RB1_RPTR_ADDR_HI,
1640                CP_RB2_RPTR_ADDR_HI
1641        };
1642        static const unsigned cp_rb_base[] = {
1643                CP_RB0_BASE,
1644                CP_RB1_BASE,
1645                CP_RB2_BASE
1646        };
1647        static const unsigned cp_rb_rptr[] = {
1648                CP_RB0_RPTR,
1649                CP_RB1_RPTR,
1650                CP_RB2_RPTR
1651        };
1652        static const unsigned cp_rb_wptr[] = {
1653                CP_RB0_WPTR,
1654                CP_RB1_WPTR,
1655                CP_RB2_WPTR
1656        };
1657        struct radeon_ring *ring;
1658        int i, r;
1659
1660        /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1661        WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1662                                 SOFT_RESET_PA |
1663                                 SOFT_RESET_SH |
1664                                 SOFT_RESET_VGT |
1665                                 SOFT_RESET_SPI |
1666                                 SOFT_RESET_SX));
1667        RREG32(GRBM_SOFT_RESET);
1668        mdelay(15);
1669        WREG32(GRBM_SOFT_RESET, 0);
1670        RREG32(GRBM_SOFT_RESET);
1671
1672        WREG32(CP_SEM_WAIT_TIMER, 0x0);
1673        WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1674
1675        /* Set the write pointer delay */
1676        WREG32(CP_RB_WPTR_DELAY, 0);
1677
1678        WREG32(CP_DEBUG, (1 << 27));
1679
1680        /* set the wb address whether it's enabled or not */
1681        WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1682        WREG32(SCRATCH_UMSK, 0xff);
1683
1684        for (i = 0; i < 3; ++i) {
1685                uint32_t rb_cntl;
1686                uint64_t addr;
1687
1688                /* Set ring buffer size */
1689                ring = &rdev->ring[ridx[i]];
1690                rb_cntl = order_base_2(ring->ring_size / 8);
1691                rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1692#ifdef __BIG_ENDIAN
1693                rb_cntl |= BUF_SWAP_32BIT;
1694#endif
1695                WREG32(cp_rb_cntl[i], rb_cntl);
1696
1697                /* set the wb address whether it's enabled or not */
1698                addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1699                WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1700                WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1701        }
1702
1703        /* set the rb base addr, this causes an internal reset of ALL rings */
1704        for (i = 0; i < 3; ++i) {
1705                ring = &rdev->ring[ridx[i]];
1706                WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1707        }
1708
1709        for (i = 0; i < 3; ++i) {
1710                /* Initialize the ring buffer's read and write pointers */
1711                ring = &rdev->ring[ridx[i]];
1712                WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1713
1714                ring->wptr = 0;
1715                WREG32(cp_rb_rptr[i], 0);
1716                WREG32(cp_rb_wptr[i], ring->wptr);
1717
1718                mdelay(1);
1719                WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1720        }
1721
1722        /* start the rings */
1723        cayman_cp_start(rdev);
1724        rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1725        rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1726        rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1727        /* this only test cp0 */
1728        r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1729        if (r) {
1730                rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1731                rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1732                rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1733                return r;
1734        }
1735
1736        if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1737                radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1738
1739        return 0;
1740}
1741
1742u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1743{
1744        u32 reset_mask = 0;
1745        u32 tmp;
1746
1747        /* GRBM_STATUS */
1748        tmp = RREG32(GRBM_STATUS);
1749        if (tmp & (PA_BUSY | SC_BUSY |
1750                   SH_BUSY | SX_BUSY |
1751                   TA_BUSY | VGT_BUSY |
1752                   DB_BUSY | CB_BUSY |
1753                   GDS_BUSY | SPI_BUSY |
1754                   IA_BUSY | IA_BUSY_NO_DMA))
1755                reset_mask |= RADEON_RESET_GFX;
1756
1757        if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1758                   CP_BUSY | CP_COHERENCY_BUSY))
1759                reset_mask |= RADEON_RESET_CP;
1760
1761        if (tmp & GRBM_EE_BUSY)
1762                reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1763
1764        /* DMA_STATUS_REG 0 */
1765        tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1766        if (!(tmp & DMA_IDLE))
1767                reset_mask |= RADEON_RESET_DMA;
1768
1769        /* DMA_STATUS_REG 1 */
1770        tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1771        if (!(tmp & DMA_IDLE))
1772                reset_mask |= RADEON_RESET_DMA1;
1773
1774        /* SRBM_STATUS2 */
1775        tmp = RREG32(SRBM_STATUS2);
1776        if (tmp & DMA_BUSY)
1777                reset_mask |= RADEON_RESET_DMA;
1778
1779        if (tmp & DMA1_BUSY)
1780                reset_mask |= RADEON_RESET_DMA1;
1781
1782        /* SRBM_STATUS */
1783        tmp = RREG32(SRBM_STATUS);
1784        if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1785                reset_mask |= RADEON_RESET_RLC;
1786
1787        if (tmp & IH_BUSY)
1788                reset_mask |= RADEON_RESET_IH;
1789
1790        if (tmp & SEM_BUSY)
1791                reset_mask |= RADEON_RESET_SEM;
1792
1793        if (tmp & GRBM_RQ_PENDING)
1794                reset_mask |= RADEON_RESET_GRBM;
1795
1796        if (tmp & VMC_BUSY)
1797                reset_mask |= RADEON_RESET_VMC;
1798
1799        if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1800                   MCC_BUSY | MCD_BUSY))
1801                reset_mask |= RADEON_RESET_MC;
1802
1803        if (evergreen_is_display_hung(rdev))
1804                reset_mask |= RADEON_RESET_DISPLAY;
1805
1806        /* VM_L2_STATUS */
1807        tmp = RREG32(VM_L2_STATUS);
1808        if (tmp & L2_BUSY)
1809                reset_mask |= RADEON_RESET_VMC;
1810
1811        /* Skip MC reset as it's mostly likely not hung, just busy */
1812        if (reset_mask & RADEON_RESET_MC) {
1813                DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1814                reset_mask &= ~RADEON_RESET_MC;
1815        }
1816
1817        return reset_mask;
1818}
1819
1820static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1821{
1822        struct evergreen_mc_save save;
1823        u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1824        u32 tmp;
1825
1826        if (reset_mask == 0)
1827                return;
1828
1829        dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1830
1831        evergreen_print_gpu_status_regs(rdev);
1832        dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1833                 RREG32(0x14F8));
1834        dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1835                 RREG32(0x14D8));
1836        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1837                 RREG32(0x14FC));
1838        dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1839                 RREG32(0x14DC));
1840
1841        /* Disable CP parsing/prefetching */
1842        WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1843
1844        if (reset_mask & RADEON_RESET_DMA) {
1845                /* dma0 */
1846                tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1847                tmp &= ~DMA_RB_ENABLE;
1848                WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1849        }
1850
1851        if (reset_mask & RADEON_RESET_DMA1) {
1852                /* dma1 */
1853                tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1854                tmp &= ~DMA_RB_ENABLE;
1855                WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1856        }
1857
1858        udelay(50);
1859
1860        evergreen_mc_stop(rdev, &save);
1861        if (evergreen_mc_wait_for_idle(rdev)) {
1862                dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1863        }
1864
1865        if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1866                grbm_soft_reset = SOFT_RESET_CB |
1867                        SOFT_RESET_DB |
1868                        SOFT_RESET_GDS |
1869                        SOFT_RESET_PA |
1870                        SOFT_RESET_SC |
1871                        SOFT_RESET_SPI |
1872                        SOFT_RESET_SH |
1873                        SOFT_RESET_SX |
1874                        SOFT_RESET_TC |
1875                        SOFT_RESET_TA |
1876                        SOFT_RESET_VGT |
1877                        SOFT_RESET_IA;
1878        }
1879
1880        if (reset_mask & RADEON_RESET_CP) {
1881                grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1882
1883                srbm_soft_reset |= SOFT_RESET_GRBM;
1884        }
1885
1886        if (reset_mask & RADEON_RESET_DMA)
1887                srbm_soft_reset |= SOFT_RESET_DMA;
1888
1889        if (reset_mask & RADEON_RESET_DMA1)
1890                srbm_soft_reset |= SOFT_RESET_DMA1;
1891
1892        if (reset_mask & RADEON_RESET_DISPLAY)
1893                srbm_soft_reset |= SOFT_RESET_DC;
1894
1895        if (reset_mask & RADEON_RESET_RLC)
1896                srbm_soft_reset |= SOFT_RESET_RLC;
1897
1898        if (reset_mask & RADEON_RESET_SEM)
1899                srbm_soft_reset |= SOFT_RESET_SEM;
1900
1901        if (reset_mask & RADEON_RESET_IH)
1902                srbm_soft_reset |= SOFT_RESET_IH;
1903
1904        if (reset_mask & RADEON_RESET_GRBM)
1905                srbm_soft_reset |= SOFT_RESET_GRBM;
1906
1907        if (reset_mask & RADEON_RESET_VMC)
1908                srbm_soft_reset |= SOFT_RESET_VMC;
1909
1910        if (!(rdev->flags & RADEON_IS_IGP)) {
1911                if (reset_mask & RADEON_RESET_MC)
1912                        srbm_soft_reset |= SOFT_RESET_MC;
1913        }
1914
1915        if (grbm_soft_reset) {
1916                tmp = RREG32(GRBM_SOFT_RESET);
1917                tmp |= grbm_soft_reset;
1918                dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1919                WREG32(GRBM_SOFT_RESET, tmp);
1920                tmp = RREG32(GRBM_SOFT_RESET);
1921
1922                udelay(50);
1923
1924                tmp &= ~grbm_soft_reset;
1925                WREG32(GRBM_SOFT_RESET, tmp);
1926                tmp = RREG32(GRBM_SOFT_RESET);
1927        }
1928
1929        if (srbm_soft_reset) {
1930                tmp = RREG32(SRBM_SOFT_RESET);
1931                tmp |= srbm_soft_reset;
1932                dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1933                WREG32(SRBM_SOFT_RESET, tmp);
1934                tmp = RREG32(SRBM_SOFT_RESET);
1935
1936                udelay(50);
1937
1938                tmp &= ~srbm_soft_reset;
1939                WREG32(SRBM_SOFT_RESET, tmp);
1940                tmp = RREG32(SRBM_SOFT_RESET);
1941        }
1942
1943        /* Wait a little for things to settle down */
1944        udelay(50);
1945
1946        evergreen_mc_resume(rdev, &save);
1947        udelay(50);
1948
1949        evergreen_print_gpu_status_regs(rdev);
1950}
1951
1952int cayman_asic_reset(struct radeon_device *rdev, bool hard)
1953{
1954        u32 reset_mask;
1955
1956        if (hard) {
1957                evergreen_gpu_pci_config_reset(rdev);
1958                return 0;
1959        }
1960
1961        reset_mask = cayman_gpu_check_soft_reset(rdev);
1962
1963        if (reset_mask)
1964                r600_set_bios_scratch_engine_hung(rdev, true);
1965
1966        cayman_gpu_soft_reset(rdev, reset_mask);
1967
1968        reset_mask = cayman_gpu_check_soft_reset(rdev);
1969
1970        if (reset_mask)
1971                evergreen_gpu_pci_config_reset(rdev);
1972
1973        r600_set_bios_scratch_engine_hung(rdev, false);
1974
1975        return 0;
1976}
1977
1978/**
1979 * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1980 *
1981 * @rdev: radeon_device pointer
1982 * @ring: radeon_ring structure holding ring information
1983 *
1984 * Check if the GFX engine is locked up.
1985 * Returns true if the engine appears to be locked up, false if not.
1986 */
1987bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1988{
1989        u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1990
1991        if (!(reset_mask & (RADEON_RESET_GFX |
1992                            RADEON_RESET_COMPUTE |
1993                            RADEON_RESET_CP))) {
1994                radeon_ring_lockup_update(rdev, ring);
1995                return false;
1996        }
1997        return radeon_ring_test_lockup(rdev, ring);
1998}
1999
2000static void cayman_uvd_init(struct radeon_device *rdev)
2001{
2002        int r;
2003
2004        if (!rdev->has_uvd)
2005                return;
2006
2007        r = radeon_uvd_init(rdev);
2008        if (r) {
2009                dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
2010                /*
2011                 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
2012                 * to early fails uvd_v2_2_resume() and thus nothing happens
2013                 * there. So it is pointless to try to go through that code
2014                 * hence why we disable uvd here.
2015                 */
2016                rdev->has_uvd = 0;
2017                return;
2018        }
2019        rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
2020        r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
2021}
2022
2023static void cayman_uvd_start(struct radeon_device *rdev)
2024{
2025        int r;
2026
2027        if (!rdev->has_uvd)
2028                return;
2029
2030        r = uvd_v2_2_resume(rdev);
2031        if (r) {
2032                dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
2033                goto error;
2034        }
2035        r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
2036        if (r) {
2037                dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
2038                goto error;
2039        }
2040        return;
2041
2042error:
2043        rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2044}
2045
2046static void cayman_uvd_resume(struct radeon_device *rdev)
2047{
2048        struct radeon_ring *ring;
2049        int r;
2050
2051        if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
2052                return;
2053
2054        ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2055        r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
2056        if (r) {
2057                dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
2058                return;
2059        }
2060        r = uvd_v1_0_init(rdev);
2061        if (r) {
2062                dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
2063                return;
2064        }
2065}
2066
2067static void cayman_vce_init(struct radeon_device *rdev)
2068{
2069        int r;
2070
2071        /* Only set for CHIP_ARUBA */
2072        if (!rdev->has_vce)
2073                return;
2074
2075        r = radeon_vce_init(rdev);
2076        if (r) {
2077                dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
2078                /*
2079                 * At this point rdev->vce.vcpu_bo is NULL which trickles down
2080                 * to early fails cayman_vce_start() and thus nothing happens
2081                 * there. So it is pointless to try to go through that code
2082                 * hence why we disable vce here.
2083                 */
2084                rdev->has_vce = 0;
2085                return;
2086        }
2087        rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
2088        r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
2089        rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
2090        r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
2091}
2092
2093static void cayman_vce_start(struct radeon_device *rdev)
2094{
2095        int r;
2096
2097        if (!rdev->has_vce)
2098                return;
2099
2100        r = radeon_vce_resume(rdev);
2101        if (r) {
2102                dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2103                goto error;
2104        }
2105        r = vce_v1_0_resume(rdev);
2106        if (r) {
2107                dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
2108                goto error;
2109        }
2110        r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
2111        if (r) {
2112                dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
2113                goto error;
2114        }
2115        r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
2116        if (r) {
2117                dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
2118                goto error;
2119        }
2120        return;
2121
2122error:
2123        rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
2124        rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
2125}
2126
2127static void cayman_vce_resume(struct radeon_device *rdev)
2128{
2129        struct radeon_ring *ring;
2130        int r;
2131
2132        if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
2133                return;
2134
2135        ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
2136        r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2137        if (r) {
2138                dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2139                return;
2140        }
2141        ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
2142        r = radeon_ring_init(rdev, ring, ring->ring_size, 0, 0x0);
2143        if (r) {
2144                dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
2145                return;
2146        }
2147        r = vce_v1_0_init(rdev);
2148        if (r) {
2149                dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
2150                return;
2151        }
2152}
2153
2154static int cayman_startup(struct radeon_device *rdev)
2155{
2156        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2157        int r;
2158
2159        /* enable pcie gen2 link */
2160        evergreen_pcie_gen2_enable(rdev);
2161        /* enable aspm */
2162        evergreen_program_aspm(rdev);
2163
2164        /* scratch needs to be initialized before MC */
2165        r = r600_vram_scratch_init(rdev);
2166        if (r)
2167                return r;
2168
2169        evergreen_mc_program(rdev);
2170
2171        if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
2172                r = ni_mc_load_microcode(rdev);
2173                if (r) {
2174                        DRM_ERROR("Failed to load MC firmware!\n");
2175                        return r;
2176                }
2177        }
2178
2179        r = cayman_pcie_gart_enable(rdev);
2180        if (r)
2181                return r;
2182        cayman_gpu_init(rdev);
2183
2184        /* allocate rlc buffers */
2185        if (rdev->flags & RADEON_IS_IGP) {
2186                rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
2187                rdev->rlc.reg_list_size =
2188                        (u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
2189                rdev->rlc.cs_data = cayman_cs_data;
2190                r = sumo_rlc_init(rdev);
2191                if (r) {
2192                        DRM_ERROR("Failed to init rlc BOs!\n");
2193                        return r;
2194                }
2195        }
2196
2197        /* allocate wb buffer */
2198        r = radeon_wb_init(rdev);
2199        if (r)
2200                return r;
2201
2202        r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
2203        if (r) {
2204                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2205                return r;
2206        }
2207
2208        cayman_uvd_start(rdev);
2209        cayman_vce_start(rdev);
2210
2211        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2212        if (r) {
2213                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2214                return r;
2215        }
2216
2217        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2218        if (r) {
2219                dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2220                return r;
2221        }
2222
2223        r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2224        if (r) {
2225                dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2226                return r;
2227        }
2228
2229        r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2230        if (r) {
2231                dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2232                return r;
2233        }
2234
2235        /* Enable IRQ */
2236        if (!rdev->irq.installed) {
2237                r = radeon_irq_kms_init(rdev);
2238                if (r)
2239                        return r;
2240        }
2241
2242        r = r600_irq_init(rdev);
2243        if (r) {
2244                DRM_ERROR("radeon: IH init failed (%d).\n", r);
2245                radeon_irq_kms_fini(rdev);
2246                return r;
2247        }
2248        evergreen_irq_set(rdev);
2249
2250        r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2251                             RADEON_CP_PACKET2);
2252        if (r)
2253                return r;
2254
2255        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2256        r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2257                             DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2258        if (r)
2259                return r;
2260
2261        ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2262        r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2263                             DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2264        if (r)
2265                return r;
2266
2267        r = cayman_cp_load_microcode(rdev);
2268        if (r)
2269                return r;
2270        r = cayman_cp_resume(rdev);
2271        if (r)
2272                return r;
2273
2274        r = cayman_dma_resume(rdev);
2275        if (r)
2276                return r;
2277
2278        cayman_uvd_resume(rdev);
2279        cayman_vce_resume(rdev);
2280
2281        r = radeon_ib_pool_init(rdev);
2282        if (r) {
2283                dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2284                return r;
2285        }
2286
2287        r = radeon_vm_manager_init(rdev);
2288        if (r) {
2289                dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2290                return r;
2291        }
2292
2293        r = radeon_audio_init(rdev);
2294        if (r)
2295                return r;
2296
2297        return 0;
2298}
2299
2300int cayman_resume(struct radeon_device *rdev)
2301{
2302        int r;
2303
2304        /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2305         * posting will perform necessary task to bring back GPU into good
2306         * shape.
2307         */
2308        /* post card */
2309        atom_asic_init(rdev->mode_info.atom_context);
2310
2311        /* init golden registers */
2312        ni_init_golden_registers(rdev);
2313
2314        if (rdev->pm.pm_method == PM_METHOD_DPM)
2315                radeon_pm_resume(rdev);
2316
2317        rdev->accel_working = true;
2318        r = cayman_startup(rdev);
2319        if (r) {
2320                DRM_ERROR("cayman startup failed on resume\n");
2321                rdev->accel_working = false;
2322                return r;
2323        }
2324        return r;
2325}
2326
2327int cayman_suspend(struct radeon_device *rdev)
2328{
2329        radeon_pm_suspend(rdev);
2330        radeon_audio_fini(rdev);
2331        radeon_vm_manager_fini(rdev);
2332        cayman_cp_enable(rdev, false);
2333        cayman_dma_stop(rdev);
2334        if (rdev->has_uvd) {
2335                uvd_v1_0_fini(rdev);
2336                radeon_uvd_suspend(rdev);
2337        }
2338        evergreen_irq_suspend(rdev);
2339        radeon_wb_disable(rdev);
2340        cayman_pcie_gart_disable(rdev);
2341        return 0;
2342}
2343
2344/* Plan is to move initialization in that function and use
2345 * helper function so that radeon_device_init pretty much
2346 * do nothing more than calling asic specific function. This
2347 * should also allow to remove a bunch of callback function
2348 * like vram_info.
2349 */
2350int cayman_init(struct radeon_device *rdev)
2351{
2352        struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2353        int r;
2354
2355        /* Read BIOS */
2356        if (!radeon_get_bios(rdev)) {
2357                if (ASIC_IS_AVIVO(rdev))
2358                        return -EINVAL;
2359        }
2360        /* Must be an ATOMBIOS */
2361        if (!rdev->is_atom_bios) {
2362                dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2363                return -EINVAL;
2364        }
2365        r = radeon_atombios_init(rdev);
2366        if (r)
2367                return r;
2368
2369        /* Post card if necessary */
2370        if (!radeon_card_posted(rdev)) {
2371                if (!rdev->bios) {
2372                        dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2373                        return -EINVAL;
2374                }
2375                DRM_INFO("GPU not posted. posting now...\n");
2376                atom_asic_init(rdev->mode_info.atom_context);
2377        }
2378        /* init golden registers */
2379        ni_init_golden_registers(rdev);
2380        /* Initialize scratch registers */
2381        r600_scratch_init(rdev);
2382        /* Initialize surface registers */
2383        radeon_surface_init(rdev);
2384        /* Initialize clocks */
2385        radeon_get_clock_info(rdev->ddev);
2386        /* Fence driver */
2387        r = radeon_fence_driver_init(rdev);
2388        if (r)
2389                return r;
2390        /* initialize memory controller */
2391        r = evergreen_mc_init(rdev);
2392        if (r)
2393                return r;
2394        /* Memory manager */
2395        r = radeon_bo_init(rdev);
2396        if (r)
2397                return r;
2398
2399        if (rdev->flags & RADEON_IS_IGP) {
2400                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2401                        r = ni_init_microcode(rdev);
2402                        if (r) {
2403                                DRM_ERROR("Failed to load firmware!\n");
2404                                return r;
2405                        }
2406                }
2407        } else {
2408                if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2409                        r = ni_init_microcode(rdev);
2410                        if (r) {
2411                                DRM_ERROR("Failed to load firmware!\n");
2412                                return r;
2413                        }
2414                }
2415        }
2416
2417        /* Initialize power management */
2418        radeon_pm_init(rdev);
2419
2420        ring->ring_obj = NULL;
2421        r600_ring_init(rdev, ring, 1024 * 1024);
2422
2423        ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2424        ring->ring_obj = NULL;
2425        r600_ring_init(rdev, ring, 64 * 1024);
2426
2427        ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2428        ring->ring_obj = NULL;
2429        r600_ring_init(rdev, ring, 64 * 1024);
2430
2431        cayman_uvd_init(rdev);
2432        cayman_vce_init(rdev);
2433
2434        rdev->ih.ring_obj = NULL;
2435        r600_ih_ring_init(rdev, 64 * 1024);
2436
2437        r = r600_pcie_gart_init(rdev);
2438        if (r)
2439                return r;
2440
2441        rdev->accel_working = true;
2442        r = cayman_startup(rdev);
2443        if (r) {
2444                dev_err(rdev->dev, "disabling GPU acceleration\n");
2445                cayman_cp_fini(rdev);
2446                cayman_dma_fini(rdev);
2447                r600_irq_fini(rdev);
2448                if (rdev->flags & RADEON_IS_IGP)
2449                        sumo_rlc_fini(rdev);
2450                radeon_wb_fini(rdev);
2451                radeon_ib_pool_fini(rdev);
2452                radeon_vm_manager_fini(rdev);
2453                radeon_irq_kms_fini(rdev);
2454                cayman_pcie_gart_fini(rdev);
2455                rdev->accel_working = false;
2456        }
2457
2458        /* Don't start up if the MC ucode is missing.
2459         * The default clocks and voltages before the MC ucode
2460         * is loaded are not suffient for advanced operations.
2461         *
2462         * We can skip this check for TN, because there is no MC
2463         * ucode.
2464         */
2465        if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2466                DRM_ERROR("radeon: MC ucode required for NI+.\n");
2467                return -EINVAL;
2468        }
2469
2470        return 0;
2471}
2472
2473void cayman_fini(struct radeon_device *rdev)
2474{
2475        radeon_pm_fini(rdev);
2476        cayman_cp_fini(rdev);
2477        cayman_dma_fini(rdev);
2478        r600_irq_fini(rdev);
2479        if (rdev->flags & RADEON_IS_IGP)
2480                sumo_rlc_fini(rdev);
2481        radeon_wb_fini(rdev);
2482        radeon_vm_manager_fini(rdev);
2483        radeon_ib_pool_fini(rdev);
2484        radeon_irq_kms_fini(rdev);
2485        uvd_v1_0_fini(rdev);
2486        radeon_uvd_fini(rdev);
2487        if (rdev->has_vce)
2488                radeon_vce_fini(rdev);
2489        cayman_pcie_gart_fini(rdev);
2490        r600_vram_scratch_fini(rdev);
2491        radeon_gem_fini(rdev);
2492        radeon_fence_driver_fini(rdev);
2493        radeon_bo_fini(rdev);
2494        radeon_atombios_fini(rdev);
2495        kfree(rdev->bios);
2496        rdev->bios = NULL;
2497}
2498
2499/*
2500 * vm
2501 */
2502int cayman_vm_init(struct radeon_device *rdev)
2503{
2504        /* number of VMs */
2505        rdev->vm_manager.nvm = 8;
2506        /* base offset of vram pages */
2507        if (rdev->flags & RADEON_IS_IGP) {
2508                u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2509                tmp <<= 22;
2510                rdev->vm_manager.vram_base_offset = tmp;
2511        } else
2512                rdev->vm_manager.vram_base_offset = 0;
2513        return 0;
2514}
2515
2516void cayman_vm_fini(struct radeon_device *rdev)
2517{
2518}
2519
2520/**
2521 * cayman_vm_decode_fault - print human readable fault info
2522 *
2523 * @rdev: radeon_device pointer
2524 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2525 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2526 *
2527 * Print human readable fault information (cayman/TN).
2528 */
2529void cayman_vm_decode_fault(struct radeon_device *rdev,
2530                            u32 status, u32 addr)
2531{
2532        u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2533        u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2534        u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2535        char *block;
2536
2537        switch (mc_id) {
2538        case 32:
2539        case 16:
2540        case 96:
2541        case 80:
2542        case 160:
2543        case 144:
2544        case 224:
2545        case 208:
2546                block = "CB";
2547                break;
2548        case 33:
2549        case 17:
2550        case 97:
2551        case 81:
2552        case 161:
2553        case 145:
2554        case 225:
2555        case 209:
2556                block = "CB_FMASK";
2557                break;
2558        case 34:
2559        case 18:
2560        case 98:
2561        case 82:
2562        case 162:
2563        case 146:
2564        case 226:
2565        case 210:
2566                block = "CB_CMASK";
2567                break;
2568        case 35:
2569        case 19:
2570        case 99:
2571        case 83:
2572        case 163:
2573        case 147:
2574        case 227:
2575        case 211:
2576                block = "CB_IMMED";
2577                break;
2578        case 36:
2579        case 20:
2580        case 100:
2581        case 84:
2582        case 164:
2583        case 148:
2584        case 228:
2585        case 212:
2586                block = "DB";
2587                break;
2588        case 37:
2589        case 21:
2590        case 101:
2591        case 85:
2592        case 165:
2593        case 149:
2594        case 229:
2595        case 213:
2596                block = "DB_HTILE";
2597                break;
2598        case 38:
2599        case 22:
2600        case 102:
2601        case 86:
2602        case 166:
2603        case 150:
2604        case 230:
2605        case 214:
2606                block = "SX";
2607                break;
2608        case 39:
2609        case 23:
2610        case 103:
2611        case 87:
2612        case 167:
2613        case 151:
2614        case 231:
2615        case 215:
2616                block = "DB_STEN";
2617                break;
2618        case 40:
2619        case 24:
2620        case 104:
2621        case 88:
2622        case 232:
2623        case 216:
2624        case 168:
2625        case 152:
2626                block = "TC_TFETCH";
2627                break;
2628        case 41:
2629        case 25:
2630        case 105:
2631        case 89:
2632        case 233:
2633        case 217:
2634        case 169:
2635        case 153:
2636                block = "TC_VFETCH";
2637                break;
2638        case 42:
2639        case 26:
2640        case 106:
2641        case 90:
2642        case 234:
2643        case 218:
2644        case 170:
2645        case 154:
2646                block = "VC";
2647                break;
2648        case 112:
2649                block = "CP";
2650                break;
2651        case 113:
2652        case 114:
2653                block = "SH";
2654                break;
2655        case 115:
2656                block = "VGT";
2657                break;
2658        case 178:
2659                block = "IH";
2660                break;
2661        case 51:
2662                block = "RLC";
2663                break;
2664        case 55:
2665                block = "DMA";
2666                break;
2667        case 56:
2668                block = "HDP";
2669                break;
2670        default:
2671                block = "unknown";
2672                break;
2673        }
2674
2675        printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2676               protections, vmid, addr,
2677               (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2678               block, mc_id);
2679}
2680
2681/**
2682 * cayman_vm_flush - vm flush using the CP
2683 *
2684 * @rdev: radeon_device pointer
2685 *
2686 * Update the page table base and flush the VM TLB
2687 * using the CP (cayman-si).
2688 */
2689void cayman_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
2690                     unsigned vm_id, uint64_t pd_addr)
2691{
2692        radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2), 0));
2693        radeon_ring_write(ring, pd_addr >> 12);
2694
2695        /* flush hdp cache */
2696        radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2697        radeon_ring_write(ring, 0x1);
2698
2699        /* bits 0-7 are the VM contexts0-7 */
2700        radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2701        radeon_ring_write(ring, 1 << vm_id);
2702
2703        /* wait for the invalidate to complete */
2704        radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
2705        radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
2706                                 WAIT_REG_MEM_ENGINE(0))); /* me */
2707        radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
2708        radeon_ring_write(ring, 0);
2709        radeon_ring_write(ring, 0); /* ref */
2710        radeon_ring_write(ring, 0); /* mask */
2711        radeon_ring_write(ring, 0x20); /* poll interval */
2712
2713        /* sync PFP to ME, otherwise we might get invalid PFP reads */
2714        radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2715        radeon_ring_write(ring, 0x0);
2716}
2717
2718int tn_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
2719{
2720        struct atom_clock_dividers dividers;
2721        int r, i;
2722
2723        r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2724                                           ecclk, false, &dividers);
2725        if (r)
2726                return r;
2727
2728        for (i = 0; i < 100; i++) {
2729                if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2730                        break;
2731                mdelay(10);
2732        }
2733        if (i == 100)
2734                return -ETIMEDOUT;
2735
2736        WREG32_P(CG_ECLK_CNTL, dividers.post_div, ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK));
2737
2738        for (i = 0; i < 100; i++) {
2739                if (RREG32(CG_ECLK_STATUS) & ECLK_STATUS)
2740                        break;
2741                mdelay(10);
2742        }
2743        if (i == 100)
2744                return -ETIMEDOUT;
2745
2746        return 0;
2747}
2748