LXR linux/drivers/gpu/host1x/hw/cdma

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Tegra host1x Command DMA
   4 *
   5 * Copyright (c) 2010-2013, NVIDIA Corporation.
   6 */
   7
   8#include <linux/slab.h>
   9#include <linux/scatterlist.h>
  10#include <linux/dma-mapping.h>
  11
  12#include "../cdma.h"
  13#include "../channel.h"
  14#include "../dev.h"
  15#include "../debug.h"
  16
  17/*
  18 * Put the restart at the end of pushbuffer memory
  19 */
  20static void push_buffer_init(struct push_buffer *pb)
  21{
  22        *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0);
  23}
  24
  25/*
  26 * Increment timedout buffer's syncpt via CPU.
  27 */
  28static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
  29                                u32 syncpt_incrs, u32 syncval, u32 nr_slots)
  30{
  31        unsigned int i;
  32
  33        for (i = 0; i < syncpt_incrs; i++)
  34                host1x_syncpt_incr(cdma->timeout.syncpt);
  35
  36        /* after CPU incr, ensure shadow is up to date */
  37        host1x_syncpt_load(cdma->timeout.syncpt);
  38}
  39
  40/*
  41 * Start channel DMA
  42 */
  43static void cdma_start(struct host1x_cdma *cdma)
  44{
  45        struct host1x_channel *ch = cdma_to_channel(cdma);
  46        u64 start, end;
  47
  48        if (cdma->running)
  49                return;
  50
  51        cdma->last_pos = cdma->push_buffer.pos;
  52        start = cdma->push_buffer.dma;
  53        end = cdma->push_buffer.size + 4;
  54
  55        host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
  56                         HOST1X_CHANNEL_DMACTRL);
  57
  58        /* set base, put and end pointer */
  59        host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
  60#if HOST1X_HW >= 6
  61        host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
  62#endif
  63        host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
  64#if HOST1X_HW >= 6
  65        host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMAPUT_HI);
  66#endif
  67        host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
  68#if HOST1X_HW >= 6
  69        host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
  70#endif
  71
  72        /* reset GET */
  73        host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
  74                         HOST1X_CHANNEL_DMACTRL_DMAGETRST |
  75                         HOST1X_CHANNEL_DMACTRL_DMAINITGET,
  76                         HOST1X_CHANNEL_DMACTRL);
  77
  78        /* start the command DMA */
  79        host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
  80
  81        cdma->running = true;
  82}
  83
  84/*
  85 * Similar to cdma_start(), but rather than starting from an idle
  86 * state (where DMA GET is set to DMA PUT), on a timeout we restore
  87 * DMA GET from an explicit value (so DMA may again be pending).
  88 */
  89static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
  90{
  91        struct host1x *host1x = cdma_to_host1x(cdma);
  92        struct host1x_channel *ch = cdma_to_channel(cdma);
  93        u64 start, end;
  94
  95        if (cdma->running)
  96                return;
  97
  98        cdma->last_pos = cdma->push_buffer.pos;
  99
 100        host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 101                         HOST1X_CHANNEL_DMACTRL);
 102
 103        start = cdma->push_buffer.dma;
 104        end = cdma->push_buffer.size + 4;
 105
 106        /* set base, end pointer (all of memory) */
 107        host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
 108#if HOST1X_HW >= 6
 109        host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
 110#endif
 111        host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
 112#if HOST1X_HW >= 6
 113        host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
 114#endif
 115
 116        /* set GET, by loading the value in PUT (then reset GET) */
 117        host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
 118        host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
 119                         HOST1X_CHANNEL_DMACTRL_DMAGETRST |
 120                         HOST1X_CHANNEL_DMACTRL_DMAINITGET,
 121                         HOST1X_CHANNEL_DMACTRL);
 122
 123        dev_dbg(host1x->dev,
 124                "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
 125                host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 126                host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 127                cdma->last_pos);
 128
 129        /* deassert GET reset and set PUT */
 130        host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 131                         HOST1X_CHANNEL_DMACTRL);
 132        host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
 133
 134        /* start the command DMA */
 135        host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
 136
 137        cdma->running = true;
 138}
 139
 140/*
 141 * Kick channel DMA into action by writing its PUT offset (if it has changed)
 142 */
 143static void cdma_flush(struct host1x_cdma *cdma)
 144{
 145        struct host1x_channel *ch = cdma_to_channel(cdma);
 146
 147        if (cdma->push_buffer.pos != cdma->last_pos) {
 148                host1x_ch_writel(ch, cdma->push_buffer.pos,
 149                                 HOST1X_CHANNEL_DMAPUT);
 150                cdma->last_pos = cdma->push_buffer.pos;
 151        }
 152}
 153
 154static void cdma_stop(struct host1x_cdma *cdma)
 155{
 156        struct host1x_channel *ch = cdma_to_channel(cdma);
 157
 158        mutex_lock(&cdma->lock);
 159
 160        if (cdma->running) {
 161                host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
 162                host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 163                                 HOST1X_CHANNEL_DMACTRL);
 164                cdma->running = false;
 165        }
 166
 167        mutex_unlock(&cdma->lock);
 168}
 169
 170static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch,
 171                                 bool stop)
 172{
 173#if HOST1X_HW >= 6
 174        host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP);
 175#else
 176        u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
 177        if (stop)
 178                cmdproc_stop |= BIT(ch->id);
 179        else
 180                cmdproc_stop &= ~BIT(ch->id);
 181        host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 182#endif
 183}
 184
 185static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch)
 186{
 187#if HOST1X_HW >= 6
 188        host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN);
 189#else
 190        host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
 191#endif
 192}
 193
 194/*
 195 * Stops both channel's command processor and CDMA immediately.
 196 * Also, tears down the channel and resets corresponding module.
 197 */
 198static void cdma_freeze(struct host1x_cdma *cdma)
 199{
 200        struct host1x *host = cdma_to_host1x(cdma);
 201        struct host1x_channel *ch = cdma_to_channel(cdma);
 202
 203        if (cdma->torndown && !cdma->running) {
 204                dev_warn(host->dev, "Already torn down\n");
 205                return;
 206        }
 207
 208        dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
 209
 210        cdma_hw_cmdproc_stop(host, ch, true);
 211
 212        dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
 213                __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 214                host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 215                cdma->last_pos);
 216
 217        host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 218                         HOST1X_CHANNEL_DMACTRL);
 219
 220        cdma_hw_teardown(host, ch);
 221
 222        cdma->running = false;
 223        cdma->torndown = true;
 224}
 225
 226static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
 227{
 228        struct host1x *host1x = cdma_to_host1x(cdma);
 229        struct host1x_channel *ch = cdma_to_channel(cdma);
 230
 231        dev_dbg(host1x->dev,
 232                "resuming channel (id %u, DMAGET restart = 0x%x)\n",
 233                ch->id, getptr);
 234
 235        cdma_hw_cmdproc_stop(host1x, ch, false);
 236
 237        cdma->torndown = false;
 238        cdma_timeout_restart(cdma, getptr);
 239}
 240
 241/*
 242 * If this timeout fires, it indicates the current sync_queue entry has
 243 * exceeded its TTL and the userctx should be timed out and remaining
 244 * submits already issued cleaned up (future submits return an error).
 245 */
 246static void cdma_timeout_handler(struct work_struct *work)
 247{
 248        u32 syncpt_val;
 249        struct host1x_cdma *cdma;
 250        struct host1x *host1x;
 251        struct host1x_channel *ch;
 252
 253        cdma = container_of(to_delayed_work(work), struct host1x_cdma,
 254                            timeout.wq);
 255        host1x = cdma_to_host1x(cdma);
 256        ch = cdma_to_channel(cdma);
 257
 258        host1x_debug_dump(cdma_to_host1x(cdma));
 259
 260        mutex_lock(&cdma->lock);
 261
 262        if (!cdma->timeout.client) {
 263                dev_dbg(host1x->dev,
 264                        "cdma_timeout: expired, but has no clientid\n");
 265                mutex_unlock(&cdma->lock);
 266                return;
 267        }
 268
 269        /* stop processing to get a clean snapshot */
 270        cdma_hw_cmdproc_stop(host1x, ch, true);
 271
 272        syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
 273
 274        /* has buffer actually completed? */
 275        if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
 276                dev_dbg(host1x->dev,
 277                        "cdma_timeout: expired, but buffer had completed\n");
 278                /* restore */
 279                cdma_hw_cmdproc_stop(host1x, ch, false);
 280                mutex_unlock(&cdma->lock);
 281                return;
 282        }
 283
 284        dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",
 285                 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
 286                 syncpt_val, cdma->timeout.syncpt_val);
 287
 288        /* stop HW, resetting channel/module */
 289        host1x_hw_cdma_freeze(host1x, cdma);
 290
 291        host1x_cdma_update_sync_queue(cdma, ch->dev);
 292        mutex_unlock(&cdma->lock);
 293}
 294
 295/*
 296 * Init timeout resources
 297 */
 298static int cdma_timeout_init(struct host1x_cdma *cdma)
 299{
 300        INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
 301        cdma->timeout.initialized = true;
 302
 303        return 0;
 304}
 305
 306/*
 307 * Clean up timeout resources
 308 */
 309static void cdma_timeout_destroy(struct host1x_cdma *cdma)
 310{
 311        if (cdma->timeout.initialized)
 312                cancel_delayed_work(&cdma->timeout.wq);
 313
 314        cdma->timeout.initialized = false;
 315}
 316
 317static const struct host1x_cdma_ops host1x_cdma_ops = {
 318        .start = cdma_start,
 319        .stop = cdma_stop,
 320        .flush = cdma_flush,
 321
 322        .timeout_init = cdma_timeout_init,
 323        .timeout_destroy = cdma_timeout_destroy,
 324        .freeze = cdma_freeze,
 325        .resume = cdma_resume,
 326        .timeout_cpu_incr = cdma_timeout_cpu_incr,
 327};
 328
 329static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
 330        .init = push_buffer_init,
 331};
 332