LXR linux/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgf100.c

   1/*
   2 * Copyright 2017 Red Hat Inc.
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice shall be included in
  12 * all copies or substantial portions of the Software.
  13 *
  14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20 * OTHER DEALINGS IN THE SOFTWARE.
  21 */
  22#include "vmm.h"
  23
  24#include <subdev/fb.h>
  25#include <subdev/ltc.h>
  26#include <subdev/timer.h>
  27
  28#include <nvif/if900d.h>
  29#include <nvif/unpack.h>
  30
  31static inline void
  32gf100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
  33                  u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
  34{
  35        u64 base = (addr >> 8) | map->type;
  36        u64 data = base;
  37
  38        if (map->ctag && !(map->next & (1ULL << 44))) {
  39                while (ptes--) {
  40                        data = base | ((map->ctag >> 1) << 44);
  41                        if (!(map->ctag++ & 1))
  42                                data |= BIT_ULL(60);
  43
  44                        VMM_WO064(pt, vmm, ptei++ * 8, data);
  45                        base += map->next;
  46                }
  47        } else {
  48                map->type += ptes * map->ctag;
  49
  50                while (ptes--) {
  51                        VMM_WO064(pt, vmm, ptei++ * 8, data);
  52                        data += map->next;
  53                }
  54        }
  55}
  56
  57void
  58gf100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
  59                  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
  60{
  61        VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
  62}
  63
  64void
  65gf100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
  66                  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
  67{
  68        if (map->page->shift == PAGE_SHIFT) {
  69                VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes);
  70                nvkm_kmap(pt->memory);
  71                while (ptes--) {
  72                        const u64 data = (*map->dma++ >> 8) | map->type;
  73                        VMM_WO064(pt, vmm, ptei++ * 8, data);
  74                        map->type += map->ctag;
  75                }
  76                nvkm_done(pt->memory);
  77                return;
  78        }
  79
  80        VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
  81}
  82
  83void
  84gf100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
  85                  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
  86{
  87        VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
  88}
  89
  90void
  91gf100_vmm_pgt_unmap(struct nvkm_vmm *vmm,
  92                    struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
  93{
  94        VMM_FO064(pt, vmm, ptei * 8, 0ULL, ptes);
  95}
  96
  97const struct nvkm_vmm_desc_func
  98gf100_vmm_pgt = {
  99        .unmap = gf100_vmm_pgt_unmap,
 100        .mem = gf100_vmm_pgt_mem,
 101        .dma = gf100_vmm_pgt_dma,
 102        .sgl = gf100_vmm_pgt_sgl,
 103};
 104
 105void
 106gf100_vmm_pgd_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
 107{
 108        struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
 109        struct nvkm_mmu_pt *pd = pgd->pt[0];
 110        struct nvkm_mmu_pt *pt;
 111        u64 data = 0;
 112
 113        if ((pt = pgt->pt[0])) {
 114                switch (nvkm_memory_target(pt->memory)) {
 115                case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 0; break;
 116                case NVKM_MEM_TARGET_HOST: data |= 2ULL << 0;
 117                        data |= BIT_ULL(35); /* VOL */
 118                        break;
 119                case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 0; break;
 120                default:
 121                        WARN_ON(1);
 122                        return;
 123                }
 124                data |= pt->addr >> 8;
 125        }
 126
 127        if ((pt = pgt->pt[1])) {
 128                switch (nvkm_memory_target(pt->memory)) {
 129                case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 32; break;
 130                case NVKM_MEM_TARGET_HOST: data |= 2ULL << 32;
 131                        data |= BIT_ULL(34); /* VOL */
 132                        break;
 133                case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 32; break;
 134                default:
 135                        WARN_ON(1);
 136                        return;
 137                }
 138                data |= pt->addr << 24;
 139        }
 140
 141        nvkm_kmap(pd->memory);
 142        VMM_WO064(pd, vmm, pdei * 8, data);
 143        nvkm_done(pd->memory);
 144}
 145
 146const struct nvkm_vmm_desc_func
 147gf100_vmm_pgd = {
 148        .unmap = gf100_vmm_pgt_unmap,
 149        .pde = gf100_vmm_pgd_pde,
 150};
 151
 152static const struct nvkm_vmm_desc
 153gf100_vmm_desc_17_12[] = {
 154        { SPT, 15, 8, 0x1000, &gf100_vmm_pgt },
 155        { PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
 156        {}
 157};
 158
 159static const struct nvkm_vmm_desc
 160gf100_vmm_desc_17_17[] = {
 161        { LPT, 10, 8, 0x1000, &gf100_vmm_pgt },
 162        { PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
 163        {}
 164};
 165
 166static const struct nvkm_vmm_desc
 167gf100_vmm_desc_16_12[] = {
 168        { SPT, 14, 8, 0x1000, &gf100_vmm_pgt },
 169        { PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
 170        {}
 171};
 172
 173static const struct nvkm_vmm_desc
 174gf100_vmm_desc_16_16[] = {
 175        { LPT, 10, 8, 0x1000, &gf100_vmm_pgt },
 176        { PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
 177        {}
 178};
 179
 180void
 181gf100_vmm_flush_(struct nvkm_vmm *vmm, int depth)
 182{
 183        struct nvkm_subdev *subdev = &vmm->mmu->subdev;
 184        struct nvkm_device *device = subdev->device;
 185        u32 type = depth << 24;
 186
 187        type = 0x00000001; /* PAGE_ALL */
 188        if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR]))
 189                type |= 0x00000004; /* HUB_ONLY */
 190
 191        mutex_lock(&subdev->mutex);
 192        /* Looks like maybe a "free flush slots" counter, the
 193         * faster you write to 0x100cbc to more it decreases.
 194         */
 195        nvkm_msec(device, 2000,
 196                if (nvkm_rd32(device, 0x100c80) & 0x00ff0000)
 197                        break;
 198        );
 199
 200        nvkm_wr32(device, 0x100cb8, vmm->pd->pt[0]->addr >> 8);
 201        nvkm_wr32(device, 0x100cbc, 0x80000000 | type);
 202
 203        /* Wait for flush to be queued? */
 204        nvkm_msec(device, 2000,
 205                if (nvkm_rd32(device, 0x100c80) & 0x00008000)
 206                        break;
 207        );
 208        mutex_unlock(&subdev->mutex);
 209}
 210
 211void
 212gf100_vmm_flush(struct nvkm_vmm *vmm, int depth)
 213{
 214        gf100_vmm_flush_(vmm, 0);
 215}
 216
 217int
 218gf100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc,
 219                struct nvkm_vmm_map *map)
 220{
 221        const enum nvkm_memory_target target = nvkm_memory_target(map->memory);
 222        const struct nvkm_vmm_page *page = map->page;
 223        const bool gm20x = page->desc->func->sparse != NULL;
 224        union {
 225                struct gf100_vmm_map_vn vn;
 226                struct gf100_vmm_map_v0 v0;
 227        } *args = argv;
 228        struct nvkm_device *device = vmm->mmu->subdev.device;
 229        struct nvkm_memory *memory = map->memory;
 230        u8  kind, priv, ro, vol;
 231        int kindn, aper, ret = -ENOSYS;
 232        const u8 *kindm;
 233
 234        map->next = (1 << page->shift) >> 8;
 235        map->type = map->ctag = 0;
 236
 237        if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
 238                vol  = !!args->v0.vol;
 239                ro   = !!args->v0.ro;
 240                priv = !!args->v0.priv;
 241                kind =   args->v0.kind;
 242        } else
 243        if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
 244                vol  = target == NVKM_MEM_TARGET_HOST;
 245                ro   = 0;
 246                priv = 0;
 247                kind = 0x00;
 248        } else {
 249                VMM_DEBUG(vmm, "args");
 250                return ret;
 251        }
 252
 253        aper = vmm->func->aper(target);
 254        if (WARN_ON(aper < 0))
 255                return aper;
 256
 257        kindm = vmm->mmu->func->kind(vmm->mmu, &kindn);
 258        if (kind >= kindn || kindm[kind] == 0xff) {
 259                VMM_DEBUG(vmm, "kind %02x", kind);
 260                return -EINVAL;
 261        }
 262
 263        if (kindm[kind] != kind) {
 264                u32 comp = (page->shift == 16 && !gm20x) ? 16 : 17;
 265                u32 tags = ALIGN(nvkm_memory_size(memory), 1 << 17) >> comp;
 266                if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) {
 267                        VMM_DEBUG(vmm, "comp %d %02x", aper, page->type);
 268                        return -EINVAL;
 269                }
 270
 271                ret = nvkm_memory_tags_get(memory, device, tags,
 272                                           nvkm_ltc_tags_clear,
 273                                           &map->tags);
 274                if (ret) {
 275                        VMM_DEBUG(vmm, "comp %d", ret);
 276                        return ret;
 277                }
 278
 279                if (map->tags->mn) {
 280                        u64 tags = map->tags->mn->offset + (map->offset >> 17);
 281                        if (page->shift == 17 || !gm20x) {
 282                                map->type |= tags << 44;
 283                                map->ctag |= 1ULL << 44;
 284                                map->next |= 1ULL << 44;
 285                        } else {
 286                                map->ctag |= tags << 1 | 1;
 287                        }
 288                } else {
 289                        kind = kindm[kind];
 290                }
 291        }
 292
 293        map->type |= BIT(0);
 294        map->type |= (u64)priv << 1;
 295        map->type |= (u64)  ro << 2;
 296        map->type |= (u64) vol << 32;
 297        map->type |= (u64)aper << 33;
 298        map->type |= (u64)kind << 36;
 299        return 0;
 300}
 301
 302int
 303gf100_vmm_aper(enum nvkm_memory_target target)
 304{
 305        switch (target) {
 306        case NVKM_MEM_TARGET_VRAM: return 0;
 307        case NVKM_MEM_TARGET_HOST: return 2;
 308        case NVKM_MEM_TARGET_NCOH: return 3;
 309        default:
 310                return -EINVAL;
 311        }
 312}
 313
 314void
 315gf100_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
 316{
 317        nvkm_fo64(inst, 0x0200, 0x00000000, 2);
 318}
 319
 320int
 321gf100_vmm_join_(struct nvkm_vmm *vmm, struct nvkm_memory *inst, u64 base)
 322{
 323        struct nvkm_mmu_pt *pd = vmm->pd->pt[0];
 324
 325        switch (nvkm_memory_target(pd->memory)) {
 326        case NVKM_MEM_TARGET_VRAM: base |= 0ULL << 0; break;
 327        case NVKM_MEM_TARGET_HOST: base |= 2ULL << 0;
 328                base |= BIT_ULL(2) /* VOL. */;
 329                break;
 330        case NVKM_MEM_TARGET_NCOH: base |= 3ULL << 0; break;
 331        default:
 332                WARN_ON(1);
 333                return -EINVAL;
 334        }
 335        base |= pd->addr;
 336
 337        nvkm_kmap(inst);
 338        nvkm_wo64(inst, 0x0200, base);
 339        nvkm_wo64(inst, 0x0208, vmm->limit - 1);
 340        nvkm_done(inst);
 341        return 0;
 342}
 343
 344int
 345gf100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
 346{
 347        return gf100_vmm_join_(vmm, inst, 0);
 348}
 349
 350static const struct nvkm_vmm_func
 351gf100_vmm_17 = {
 352        .join = gf100_vmm_join,
 353        .part = gf100_vmm_part,
 354        .aper = gf100_vmm_aper,
 355        .valid = gf100_vmm_valid,
 356        .flush = gf100_vmm_flush,
 357        .page = {
 358                { 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC },
 359                { 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx },
 360                {}
 361        }
 362};
 363
 364static const struct nvkm_vmm_func
 365gf100_vmm_16 = {
 366        .join = gf100_vmm_join,
 367        .part = gf100_vmm_part,
 368        .aper = gf100_vmm_aper,
 369        .valid = gf100_vmm_valid,
 370        .flush = gf100_vmm_flush,
 371        .page = {
 372                { 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC },
 373                { 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx },
 374                {}
 375        }
 376};
 377
 378int
 379gf100_vmm_new_(const struct nvkm_vmm_func *func_16,
 380               const struct nvkm_vmm_func *func_17,
 381               struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
 382               struct lock_class_key *key, const char *name,
 383               struct nvkm_vmm **pvmm)
 384{
 385        switch (mmu->subdev.device->fb->page) {
 386        case 16: return nv04_vmm_new_(func_16, mmu, 0, addr, size,
 387                                      argv, argc, key, name, pvmm);
 388        case 17: return nv04_vmm_new_(func_17, mmu, 0, addr, size,
 389                                      argv, argc, key, name, pvmm);
 390        default:
 391                WARN_ON(1);
 392                return -EINVAL;
 393        }
 394}
 395
 396int
 397gf100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
 398              struct lock_class_key *key, const char *name,
 399              struct nvkm_vmm **pvmm)
 400{
 401        return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, addr,
 402                              size, argv, argc, key, name, pvmm);
 403}
 404