LXR linux/drivers/gpu/drm/i915/gt/intel

   1/*
   2 * Copyright (c) 2015 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions: *
  10 * The above copyright notice and this permission notice (including the next
  11 * paragraph) shall be included in all copies or substantial portions of the
  12 * Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20 * SOFTWARE.
  21 */
  22
  23#include "i915_drv.h"
  24
  25#include "intel_engine.h"
  26#include "intel_gt.h"
  27#include "intel_mocs.h"
  28#include "intel_lrc.h"
  29#include "intel_ring.h"
  30
  31/* structures required */
  32struct drm_i915_mocs_entry {
  33        u32 control_value;
  34        u16 l3cc_value;
  35        u16 used;
  36};
  37
  38struct drm_i915_mocs_table {
  39        unsigned int size;
  40        unsigned int n_entries;
  41        const struct drm_i915_mocs_entry *table;
  42};
  43
  44/* Defines for the tables (XXX_MOCS_0 - XXX_MOCS_63) */
  45#define _LE_CACHEABILITY(value) ((value) << 0)
  46#define _LE_TGT_CACHE(value)    ((value) << 2)
  47#define LE_LRUM(value)          ((value) << 4)
  48#define LE_AOM(value)           ((value) << 6)
  49#define LE_RSC(value)           ((value) << 7)
  50#define LE_SCC(value)           ((value) << 8)
  51#define LE_PFM(value)           ((value) << 11)
  52#define LE_SCF(value)           ((value) << 14)
  53#define LE_COS(value)           ((value) << 15)
  54#define LE_SSE(value)           ((value) << 17)
  55
  56/* Defines for the tables (LNCFMOCS0 - LNCFMOCS31) - two entries per word */
  57#define L3_ESC(value)           ((value) << 0)
  58#define L3_SCC(value)           ((value) << 1)
  59#define _L3_CACHEABILITY(value) ((value) << 4)
  60
  61/* Helper defines */
  62#define GEN9_NUM_MOCS_ENTRIES   62  /* 62 out of 64 - 63 & 64 are reserved. */
  63#define GEN11_NUM_MOCS_ENTRIES  64  /* 63-64 are reserved, but configured. */
  64
  65/* (e)LLC caching options */
  66/*
  67 * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means
  68 * the same as LE_UC
  69 */
  70#define LE_0_PAGETABLE          _LE_CACHEABILITY(0)
  71#define LE_1_UC                 _LE_CACHEABILITY(1)
  72#define LE_2_WT                 _LE_CACHEABILITY(2)
  73#define LE_3_WB                 _LE_CACHEABILITY(3)
  74
  75/* Target cache */
  76#define LE_TC_0_PAGETABLE       _LE_TGT_CACHE(0)
  77#define LE_TC_1_LLC             _LE_TGT_CACHE(1)
  78#define LE_TC_2_LLC_ELLC        _LE_TGT_CACHE(2)
  79#define LE_TC_3_LLC_ELLC_ALT    _LE_TGT_CACHE(3)
  80
  81/* L3 caching options */
  82#define L3_0_DIRECT             _L3_CACHEABILITY(0)
  83#define L3_1_UC                 _L3_CACHEABILITY(1)
  84#define L3_2_RESERVED           _L3_CACHEABILITY(2)
  85#define L3_3_WB                 _L3_CACHEABILITY(3)
  86
  87#define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \
  88        [__idx] = { \
  89                .control_value = __control_value, \
  90                .l3cc_value = __l3cc_value, \
  91                .used = 1, \
  92        }
  93
  94/*
  95 * MOCS tables
  96 *
  97 * These are the MOCS tables that are programmed across all the rings.
  98 * The control value is programmed to all the rings that support the
  99 * MOCS registers. While the l3cc_values are only programmed to the
 100 * LNCFCMOCS0 - LNCFCMOCS32 registers.
 101 *
 102 * These tables are intended to be kept reasonably consistent across
 103 * HW platforms, and for ICL+, be identical across OSes. To achieve
 104 * that, for Icelake and above, list of entries is published as part
 105 * of bspec.
 106 *
 107 * Entries not part of the following tables are undefined as far as
 108 * userspace is concerned and shouldn't be relied upon.  For Gen < 12
 109 * they will be initialized to PTE. Gen >= 12 onwards don't have a setting for
 110 * PTE and will be initialized to an invalid value.
 111 *
 112 * The last two entries are reserved by the hardware. For ICL+ they
 113 * should be initialized according to bspec and never used, for older
 114 * platforms they should never be written to.
 115 *
 116 * NOTE: These tables are part of bspec and defined as part of hardware
 117 *       interface for ICL+. For older platforms, they are part of kernel
 118 *       ABI. It is expected that, for specific hardware platform, existing
 119 *       entries will remain constant and the table will only be updated by
 120 *       adding new entries, filling unused positions.
 121 */
 122#define GEN9_MOCS_ENTRIES \
 123        MOCS_ENTRY(I915_MOCS_UNCACHED, \
 124                   LE_1_UC | LE_TC_2_LLC_ELLC, \
 125                   L3_1_UC), \
 126        MOCS_ENTRY(I915_MOCS_PTE, \
 127                   LE_0_PAGETABLE | LE_TC_2_LLC_ELLC | LE_LRUM(3), \
 128                   L3_3_WB)
 129
 130static const struct drm_i915_mocs_entry skl_mocs_table[] = {
 131        GEN9_MOCS_ENTRIES,
 132        MOCS_ENTRY(I915_MOCS_CACHED,
 133                   LE_3_WB | LE_TC_2_LLC_ELLC | LE_LRUM(3),
 134                   L3_3_WB)
 135};
 136
 137/* NOTE: the LE_TGT_CACHE is not used on Broxton */
 138static const struct drm_i915_mocs_entry broxton_mocs_table[] = {
 139        GEN9_MOCS_ENTRIES,
 140        MOCS_ENTRY(I915_MOCS_CACHED,
 141                   LE_1_UC | LE_TC_2_LLC_ELLC | LE_LRUM(3),
 142                   L3_3_WB)
 143};
 144
 145#define GEN11_MOCS_ENTRIES \
 146        /* Entries 0 and 1 are defined per-platform */ \
 147        /* Base - L3 + LLC */ \
 148        MOCS_ENTRY(2, \
 149                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
 150                   L3_3_WB), \
 151        /* Base - Uncached */ \
 152        MOCS_ENTRY(3, \
 153                   LE_1_UC | LE_TC_1_LLC, \
 154                   L3_1_UC), \
 155        /* Base - L3 */ \
 156        MOCS_ENTRY(4, \
 157                   LE_1_UC | LE_TC_1_LLC, \
 158                   L3_3_WB), \
 159        /* Base - LLC */ \
 160        MOCS_ENTRY(5, \
 161                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
 162                   L3_1_UC), \
 163        /* Age 0 - LLC */ \
 164        MOCS_ENTRY(6, \
 165                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \
 166                   L3_1_UC), \
 167        /* Age 0 - L3 + LLC */ \
 168        MOCS_ENTRY(7, \
 169                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \
 170                   L3_3_WB), \
 171        /* Age: Don't Chg. - LLC */ \
 172        MOCS_ENTRY(8, \
 173                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \
 174                   L3_1_UC), \
 175        /* Age: Don't Chg. - L3 + LLC */ \
 176        MOCS_ENTRY(9, \
 177                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \
 178                   L3_3_WB), \
 179        /* No AOM - LLC */ \
 180        MOCS_ENTRY(10, \
 181                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \
 182                   L3_1_UC), \
 183        /* No AOM - L3 + LLC */ \
 184        MOCS_ENTRY(11, \
 185                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \
 186                   L3_3_WB), \
 187        /* No AOM; Age 0 - LLC */ \
 188        MOCS_ENTRY(12, \
 189                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \
 190                   L3_1_UC), \
 191        /* No AOM; Age 0 - L3 + LLC */ \
 192        MOCS_ENTRY(13, \
 193                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \
 194                   L3_3_WB), \
 195        /* No AOM; Age:DC - LLC */ \
 196        MOCS_ENTRY(14, \
 197                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \
 198                   L3_1_UC), \
 199        /* No AOM; Age:DC - L3 + LLC */ \
 200        MOCS_ENTRY(15, \
 201                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \
 202                   L3_3_WB), \
 203        /* Self-Snoop - L3 + LLC */ \
 204        MOCS_ENTRY(18, \
 205                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), \
 206                   L3_3_WB), \
 207        /* Skip Caching - L3 + LLC(12.5%) */ \
 208        MOCS_ENTRY(19, \
 209                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), \
 210                   L3_3_WB), \
 211        /* Skip Caching - L3 + LLC(25%) */ \
 212        MOCS_ENTRY(20, \
 213                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), \
 214                   L3_3_WB), \
 215        /* Skip Caching - L3 + LLC(50%) */ \
 216        MOCS_ENTRY(21, \
 217                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), \
 218                   L3_3_WB), \
 219        /* Skip Caching - L3 + LLC(75%) */ \
 220        MOCS_ENTRY(22, \
 221                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), \
 222                   L3_3_WB), \
 223        /* Skip Caching - L3 + LLC(87.5%) */ \
 224        MOCS_ENTRY(23, \
 225                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), \
 226                   L3_3_WB), \
 227        /* HW Reserved - SW program but never use */ \
 228        MOCS_ENTRY(62, \
 229                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
 230                   L3_1_UC), \
 231        /* HW Reserved - SW program but never use */ \
 232        MOCS_ENTRY(63, \
 233                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
 234                   L3_1_UC)
 235
 236static const struct drm_i915_mocs_entry tgl_mocs_table[] = {
 237        /* Base - Error (Reserved for Non-Use) */
 238        MOCS_ENTRY(0, 0x0, 0x0),
 239        /* Base - Reserved */
 240        MOCS_ENTRY(1, 0x0, 0x0),
 241
 242        GEN11_MOCS_ENTRIES,
 243
 244        /* Implicitly enable L1 - HDC:L1 + L3 + LLC */
 245        MOCS_ENTRY(48,
 246                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
 247                   L3_3_WB),
 248        /* Implicitly enable L1 - HDC:L1 + L3 */
 249        MOCS_ENTRY(49,
 250                   LE_1_UC | LE_TC_1_LLC,
 251                   L3_3_WB),
 252        /* Implicitly enable L1 - HDC:L1 + LLC */
 253        MOCS_ENTRY(50,
 254                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
 255                   L3_1_UC),
 256        /* Implicitly enable L1 - HDC:L1 */
 257        MOCS_ENTRY(51,
 258                   LE_1_UC | LE_TC_1_LLC,
 259                   L3_1_UC),
 260        /* HW Special Case (CCS) */
 261        MOCS_ENTRY(60,
 262                   LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
 263                   L3_1_UC),
 264        /* HW Special Case (Displayable) */
 265        MOCS_ENTRY(61,
 266                   LE_1_UC | LE_TC_1_LLC,
 267                   L3_3_WB),
 268};
 269
 270static const struct drm_i915_mocs_entry icl_mocs_table[] = {
 271        /* Base - Uncached (Deprecated) */
 272        MOCS_ENTRY(I915_MOCS_UNCACHED,
 273                   LE_1_UC | LE_TC_1_LLC,
 274                   L3_1_UC),
 275        /* Base - L3 + LeCC:PAT (Deprecated) */
 276        MOCS_ENTRY(I915_MOCS_PTE,
 277                   LE_0_PAGETABLE | LE_TC_1_LLC,
 278                   L3_3_WB),
 279
 280        GEN11_MOCS_ENTRIES
 281};
 282
 283enum {
 284        HAS_GLOBAL_MOCS = BIT(0),
 285        HAS_ENGINE_MOCS = BIT(1),
 286        HAS_RENDER_L3CC = BIT(2),
 287};
 288
 289static bool has_l3cc(const struct drm_i915_private *i915)
 290{
 291        return true;
 292}
 293
 294static bool has_global_mocs(const struct drm_i915_private *i915)
 295{
 296        return HAS_GLOBAL_MOCS_REGISTERS(i915);
 297}
 298
 299static bool has_mocs(const struct drm_i915_private *i915)
 300{
 301        return !IS_DGFX(i915);
 302}
 303
 304static unsigned int get_mocs_settings(const struct drm_i915_private *i915,
 305                                      struct drm_i915_mocs_table *table)
 306{
 307        unsigned int flags;
 308
 309        if (INTEL_GEN(i915) >= 12) {
 310                table->size  = ARRAY_SIZE(tgl_mocs_table);
 311                table->table = tgl_mocs_table;
 312                table->n_entries = GEN11_NUM_MOCS_ENTRIES;
 313        } else if (IS_GEN(i915, 11)) {
 314                table->size  = ARRAY_SIZE(icl_mocs_table);
 315                table->table = icl_mocs_table;
 316                table->n_entries = GEN11_NUM_MOCS_ENTRIES;
 317        } else if (IS_GEN9_BC(i915) || IS_CANNONLAKE(i915)) {
 318                table->size  = ARRAY_SIZE(skl_mocs_table);
 319                table->n_entries = GEN9_NUM_MOCS_ENTRIES;
 320                table->table = skl_mocs_table;
 321        } else if (IS_GEN9_LP(i915)) {
 322                table->size  = ARRAY_SIZE(broxton_mocs_table);
 323                table->n_entries = GEN9_NUM_MOCS_ENTRIES;
 324                table->table = broxton_mocs_table;
 325        } else {
 326                drm_WARN_ONCE(&i915->drm, INTEL_GEN(i915) >= 9,
 327                              "Platform that should have a MOCS table does not.\n");
 328                return 0;
 329        }
 330
 331        if (GEM_DEBUG_WARN_ON(table->size > table->n_entries))
 332                return 0;
 333
 334        /* WaDisableSkipCaching:skl,bxt,kbl,glk */
 335        if (IS_GEN(i915, 9)) {
 336                int i;
 337
 338                for (i = 0; i < table->size; i++)
 339                        if (GEM_DEBUG_WARN_ON(table->table[i].l3cc_value &
 340                                              (L3_ESC(1) | L3_SCC(0x7))))
 341                                return 0;
 342        }
 343
 344        flags = 0;
 345        if (has_mocs(i915)) {
 346                if (has_global_mocs(i915))
 347                        flags |= HAS_GLOBAL_MOCS;
 348                else
 349                        flags |= HAS_ENGINE_MOCS;
 350        }
 351        if (has_l3cc(i915))
 352                flags |= HAS_RENDER_L3CC;
 353
 354        return flags;
 355}
 356
 357/*
 358 * Get control_value from MOCS entry taking into account when it's not used:
 359 * I915_MOCS_PTE's value is returned in this case.
 360 */
 361static u32 get_entry_control(const struct drm_i915_mocs_table *table,
 362                             unsigned int index)
 363{
 364        if (index < table->size && table->table[index].used)
 365                return table->table[index].control_value;
 366
 367        return table->table[I915_MOCS_PTE].control_value;
 368}
 369
 370#define for_each_mocs(mocs, t, i) \
 371        for (i = 0; \
 372             i < (t)->n_entries ? (mocs = get_entry_control((t), i)), 1 : 0;\
 373             i++)
 374
 375static void __init_mocs_table(struct intel_uncore *uncore,
 376                              const struct drm_i915_mocs_table *table,
 377                              u32 addr)
 378{
 379        unsigned int i;
 380        u32 mocs;
 381
 382        for_each_mocs(mocs, table, i)
 383                intel_uncore_write_fw(uncore, _MMIO(addr + i * 4), mocs);
 384}
 385
 386static u32 mocs_offset(const struct intel_engine_cs *engine)
 387{
 388        static const u32 offset[] = {
 389                [RCS0]  =  __GEN9_RCS0_MOCS0,
 390                [VCS0]  =  __GEN9_VCS0_MOCS0,
 391                [VCS1]  =  __GEN9_VCS1_MOCS0,
 392                [VECS0] =  __GEN9_VECS0_MOCS0,
 393                [BCS0]  =  __GEN9_BCS0_MOCS0,
 394                [VCS2]  = __GEN11_VCS2_MOCS0,
 395        };
 396
 397        GEM_BUG_ON(engine->id >= ARRAY_SIZE(offset));
 398        return offset[engine->id];
 399}
 400
 401static void init_mocs_table(struct intel_engine_cs *engine,
 402                            const struct drm_i915_mocs_table *table)
 403{
 404        __init_mocs_table(engine->uncore, table, mocs_offset(engine));
 405}
 406
 407/*
 408 * Get l3cc_value from MOCS entry taking into account when it's not used:
 409 * I915_MOCS_PTE's value is returned in this case.
 410 */
 411static u16 get_entry_l3cc(const struct drm_i915_mocs_table *table,
 412                          unsigned int index)
 413{
 414        if (index < table->size && table->table[index].used)
 415                return table->table[index].l3cc_value;
 416
 417        return table->table[I915_MOCS_PTE].l3cc_value;
 418}
 419
 420static inline u32 l3cc_combine(u16 low, u16 high)
 421{
 422        return low | (u32)high << 16;
 423}
 424
 425#define for_each_l3cc(l3cc, t, i) \
 426        for (i = 0; \
 427             i < ((t)->n_entries + 1) / 2 ? \
 428             (l3cc = l3cc_combine(get_entry_l3cc((t), 2 * i), \
 429                                  get_entry_l3cc((t), 2 * i + 1))), 1 : \
 430             0; \
 431             i++)
 432
 433static void init_l3cc_table(struct intel_engine_cs *engine,
 434                            const struct drm_i915_mocs_table *table)
 435{
 436        struct intel_uncore *uncore = engine->uncore;
 437        unsigned int i;
 438        u32 l3cc;
 439
 440        for_each_l3cc(l3cc, table, i)
 441                intel_uncore_write_fw(uncore, GEN9_LNCFCMOCS(i), l3cc);
 442}
 443
 444void intel_mocs_init_engine(struct intel_engine_cs *engine)
 445{
 446        struct drm_i915_mocs_table table;
 447        unsigned int flags;
 448
 449        /* Called under a blanket forcewake */
 450        assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);
 451
 452        flags = get_mocs_settings(engine->i915, &table);
 453        if (!flags)
 454                return;
 455
 456        /* Platforms with global MOCS do not need per-engine initialization. */
 457        if (flags & HAS_ENGINE_MOCS)
 458                init_mocs_table(engine, &table);
 459
 460        if (flags & HAS_RENDER_L3CC && engine->class == RENDER_CLASS)
 461                init_l3cc_table(engine, &table);
 462}
 463
 464static u32 global_mocs_offset(void)
 465{
 466        return i915_mmio_reg_offset(GEN12_GLOBAL_MOCS(0));
 467}
 468
 469void intel_mocs_init(struct intel_gt *gt)
 470{
 471        struct drm_i915_mocs_table table;
 472        unsigned int flags;
 473
 474        /*
 475         * LLC and eDRAM control values are not applicable to dgfx
 476         */
 477        flags = get_mocs_settings(gt->i915, &table);
 478        if (flags & HAS_GLOBAL_MOCS)
 479                __init_mocs_table(gt->uncore, &table, global_mocs_offset());
 480}
 481
 482#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 483#include "selftest_mocs.c"
 484#endif
 485