LXR linux/drivers/gpu/drm/panfrost/panfrost

   1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright 2019 Collabora Ltd */
   3
   4#include <drm/drm_file.h>
   5#include <drm/drm_gem_shmem_helper.h>
   6#include <drm/panfrost_drm.h>
   7#include <linux/completion.h>
   8#include <linux/iopoll.h>
   9#include <linux/pm_runtime.h>
  10#include <linux/slab.h>
  11#include <linux/uaccess.h>
  12
  13#include "panfrost_device.h"
  14#include "panfrost_features.h"
  15#include "panfrost_gem.h"
  16#include "panfrost_issues.h"
  17#include "panfrost_job.h"
  18#include "panfrost_mmu.h"
  19#include "panfrost_perfcnt.h"
  20#include "panfrost_regs.h"
  21
  22#define COUNTERS_PER_BLOCK              64
  23#define BYTES_PER_COUNTER               4
  24#define BLOCKS_PER_COREGROUP            8
  25#define V4_SHADERS_PER_COREGROUP        4
  26
  27struct panfrost_perfcnt {
  28        struct panfrost_gem_mapping *mapping;
  29        size_t bosize;
  30        void *buf;
  31        struct panfrost_file_priv *user;
  32        struct mutex lock;
  33        struct completion dump_comp;
  34};
  35
  36void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev)
  37{
  38        complete(&pfdev->perfcnt->dump_comp);
  39}
  40
  41void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev)
  42{
  43        gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_CACHES);
  44}
  45
  46static int panfrost_perfcnt_dump_locked(struct panfrost_device *pfdev)
  47{
  48        u64 gpuva;
  49        int ret;
  50
  51        reinit_completion(&pfdev->perfcnt->dump_comp);
  52        gpuva = pfdev->perfcnt->mapping->mmnode.start << PAGE_SHIFT;
  53        gpu_write(pfdev, GPU_PERFCNT_BASE_LO, gpuva);
  54        gpu_write(pfdev, GPU_PERFCNT_BASE_HI, gpuva >> 32);
  55        gpu_write(pfdev, GPU_INT_CLEAR,
  56                  GPU_IRQ_CLEAN_CACHES_COMPLETED |
  57                  GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
  58        gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_SAMPLE);
  59        ret = wait_for_completion_interruptible_timeout(&pfdev->perfcnt->dump_comp,
  60                                                        msecs_to_jiffies(1000));
  61        if (!ret)
  62                ret = -ETIMEDOUT;
  63        else if (ret > 0)
  64                ret = 0;
  65
  66        return ret;
  67}
  68
  69static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev,
  70                                          struct drm_file *file_priv,
  71                                          unsigned int counterset)
  72{
  73        struct panfrost_file_priv *user = file_priv->driver_priv;
  74        struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
  75        struct drm_gem_shmem_object *bo;
  76        u32 cfg, as;
  77        int ret;
  78
  79        if (user == perfcnt->user)
  80                return 0;
  81        else if (perfcnt->user)
  82                return -EBUSY;
  83
  84        ret = pm_runtime_get_sync(pfdev->dev);
  85        if (ret < 0)
  86                goto err_put_pm;
  87
  88        bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize);
  89        if (IS_ERR(bo)) {
  90                ret = PTR_ERR(bo);
  91                goto err_put_pm;
  92        }
  93
  94        /* Map the perfcnt buf in the address space attached to file_priv. */
  95        ret = panfrost_gem_open(&bo->base, file_priv);
  96        if (ret)
  97                goto err_put_bo;
  98
  99        perfcnt->mapping = panfrost_gem_mapping_get(to_panfrost_bo(&bo->base),
 100                                                    user);
 101        if (!perfcnt->mapping) {
 102                ret = -EINVAL;
 103                goto err_close_bo;
 104        }
 105
 106        perfcnt->buf = drm_gem_shmem_vmap(&bo->base);
 107        if (IS_ERR(perfcnt->buf)) {
 108                ret = PTR_ERR(perfcnt->buf);
 109                goto err_put_mapping;
 110        }
 111
 112        /*
 113         * Invalidate the cache and clear the counters to start from a fresh
 114         * state.
 115         */
 116        reinit_completion(&pfdev->perfcnt->dump_comp);
 117        gpu_write(pfdev, GPU_INT_CLEAR,
 118                  GPU_IRQ_CLEAN_CACHES_COMPLETED |
 119                  GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
 120        gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_CLEAR);
 121        gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_INV_CACHES);
 122        ret = wait_for_completion_timeout(&pfdev->perfcnt->dump_comp,
 123                                          msecs_to_jiffies(1000));
 124        if (!ret) {
 125                ret = -ETIMEDOUT;
 126                goto err_vunmap;
 127        }
 128
 129        perfcnt->user = user;
 130
 131        as = panfrost_mmu_as_get(pfdev, perfcnt->mapping->mmu);
 132        cfg = GPU_PERFCNT_CFG_AS(as) |
 133              GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL);
 134
 135        /*
 136         * Bifrost GPUs have 2 set of counters, but we're only interested by
 137         * the first one for now.
 138         */
 139        if (panfrost_model_is_bifrost(pfdev))
 140                cfg |= GPU_PERFCNT_CFG_SETSEL(counterset);
 141
 142        gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0xffffffff);
 143        gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0xffffffff);
 144        gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0xffffffff);
 145
 146        /*
 147         * Due to PRLAM-8186 we need to disable the Tiler before we enable HW
 148         * counters.
 149         */
 150        if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
 151                gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
 152        else
 153                gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
 154
 155        gpu_write(pfdev, GPU_PERFCNT_CFG, cfg);
 156
 157        if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
 158                gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
 159
 160        /* The BO ref is retained by the mapping. */
 161        drm_gem_object_put(&bo->base);
 162
 163        return 0;
 164
 165err_vunmap:
 166        drm_gem_shmem_vunmap(&bo->base, perfcnt->buf);
 167err_put_mapping:
 168        panfrost_gem_mapping_put(perfcnt->mapping);
 169err_close_bo:
 170        panfrost_gem_close(&bo->base, file_priv);
 171err_put_bo:
 172        drm_gem_object_put(&bo->base);
 173err_put_pm:
 174        pm_runtime_put(pfdev->dev);
 175        return ret;
 176}
 177
 178static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev,
 179                                           struct drm_file *file_priv)
 180{
 181        struct panfrost_file_priv *user = file_priv->driver_priv;
 182        struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
 183
 184        if (user != perfcnt->user)
 185                return -EINVAL;
 186
 187        gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0x0);
 188        gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0x0);
 189        gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0x0);
 190        gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
 191        gpu_write(pfdev, GPU_PERFCNT_CFG,
 192                  GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
 193
 194        perfcnt->user = NULL;
 195        drm_gem_shmem_vunmap(&perfcnt->mapping->obj->base.base, perfcnt->buf);
 196        perfcnt->buf = NULL;
 197        panfrost_gem_close(&perfcnt->mapping->obj->base.base, file_priv);
 198        panfrost_mmu_as_put(pfdev, perfcnt->mapping->mmu);
 199        panfrost_gem_mapping_put(perfcnt->mapping);
 200        perfcnt->mapping = NULL;
 201        pm_runtime_mark_last_busy(pfdev->dev);
 202        pm_runtime_put_autosuspend(pfdev->dev);
 203
 204        return 0;
 205}
 206
 207int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data,
 208                                  struct drm_file *file_priv)
 209{
 210        struct panfrost_device *pfdev = dev->dev_private;
 211        struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
 212        struct drm_panfrost_perfcnt_enable *req = data;
 213        int ret;
 214
 215        ret = panfrost_unstable_ioctl_check();
 216        if (ret)
 217                return ret;
 218
 219        /* Only Bifrost GPUs have 2 set of counters. */
 220        if (req->counterset > (panfrost_model_is_bifrost(pfdev) ? 1 : 0))
 221                return -EINVAL;
 222
 223        mutex_lock(&perfcnt->lock);
 224        if (req->enable)
 225                ret = panfrost_perfcnt_enable_locked(pfdev, file_priv,
 226                                                     req->counterset);
 227        else
 228                ret = panfrost_perfcnt_disable_locked(pfdev, file_priv);
 229        mutex_unlock(&perfcnt->lock);
 230
 231        return ret;
 232}
 233
 234int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data,
 235                                struct drm_file *file_priv)
 236{
 237        struct panfrost_device *pfdev = dev->dev_private;
 238        struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
 239        struct drm_panfrost_perfcnt_dump *req = data;
 240        void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr;
 241        int ret;
 242
 243        ret = panfrost_unstable_ioctl_check();
 244        if (ret)
 245                return ret;
 246
 247        mutex_lock(&perfcnt->lock);
 248        if (perfcnt->user != file_priv->driver_priv) {
 249                ret = -EINVAL;
 250                goto out;
 251        }
 252
 253        ret = panfrost_perfcnt_dump_locked(pfdev);
 254        if (ret)
 255                goto out;
 256
 257        if (copy_to_user(user_ptr, perfcnt->buf, perfcnt->bosize))
 258                ret = -EFAULT;
 259
 260out:
 261        mutex_unlock(&perfcnt->lock);
 262
 263        return ret;
 264}
 265
 266void panfrost_perfcnt_close(struct drm_file *file_priv)
 267{
 268        struct panfrost_file_priv *pfile = file_priv->driver_priv;
 269        struct panfrost_device *pfdev = pfile->pfdev;
 270        struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
 271
 272        pm_runtime_get_sync(pfdev->dev);
 273        mutex_lock(&perfcnt->lock);
 274        if (perfcnt->user == pfile)
 275                panfrost_perfcnt_disable_locked(pfdev, file_priv);
 276        mutex_unlock(&perfcnt->lock);
 277        pm_runtime_mark_last_busy(pfdev->dev);
 278        pm_runtime_put_autosuspend(pfdev->dev);
 279}
 280
 281int panfrost_perfcnt_init(struct panfrost_device *pfdev)
 282{
 283        struct panfrost_perfcnt *perfcnt;
 284        size_t size;
 285
 286        if (panfrost_has_hw_feature(pfdev, HW_FEATURE_V4)) {
 287                unsigned int ncoregroups;
 288
 289                ncoregroups = hweight64(pfdev->features.l2_present);
 290                size = ncoregroups * BLOCKS_PER_COREGROUP *
 291                       COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
 292        } else {
 293                unsigned int nl2c, ncores;
 294
 295                /*
 296                 * TODO: define a macro to extract the number of l2 caches from
 297                 * mem_features.
 298                 */
 299                nl2c = ((pfdev->features.mem_features >> 8) & GENMASK(3, 0)) + 1;
 300
 301                /*
 302                 * shader_present might be sparse, but the counters layout
 303                 * forces to dump unused regions too, hence the fls64() call
 304                 * instead of hweight64().
 305                 */
 306                ncores = fls64(pfdev->features.shader_present);
 307
 308                /*
 309                 * There's always one JM and one Tiler block, hence the '+ 2'
 310                 * here.
 311                 */
 312                size = (nl2c + ncores + 2) *
 313                       COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
 314        }
 315
 316        perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL);
 317        if (!perfcnt)
 318                return -ENOMEM;
 319
 320        perfcnt->bosize = size;
 321
 322        /* Start with everything disabled. */
 323        gpu_write(pfdev, GPU_PERFCNT_CFG,
 324                  GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
 325        gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
 326        gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
 327        gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
 328        gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
 329
 330        init_completion(&perfcnt->dump_comp);
 331        mutex_init(&perfcnt->lock);
 332        pfdev->perfcnt = perfcnt;
 333
 334        return 0;
 335}
 336
 337void panfrost_perfcnt_fini(struct panfrost_device *pfdev)
 338{
 339        /* Disable everything before leaving. */
 340        gpu_write(pfdev, GPU_PERFCNT_CFG,
 341                  GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
 342        gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
 343        gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
 344        gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
 345        gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
 346}
 347