LXR linux/drivers/gpu/host1x/hw/cdma

   1/*
   2 * Tegra host1x Command DMA
   3 *
   4 * Copyright (c) 2010-2013, NVIDIA Corporation.
   5 *
   6 * This program is free software; you can redistribute it and/or modify it
   7 * under the terms and conditions of the GNU General Public License,
   8 * version 2, as published by the Free Software Foundation.
   9 *
  10 * This program is distributed in the hope it will be useful, but WITHOUT
  11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 * more details.
  14 *
  15 * You should have received a copy of the GNU General Public License
  16 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19#include <linux/slab.h>
  20#include <linux/scatterlist.h>
  21#include <linux/dma-mapping.h>
  22
  23#include "../cdma.h"
  24#include "../channel.h"
  25#include "../dev.h"
  26#include "../debug.h"
  27
  28/*
  29 * Put the restart at the end of pushbuffer memory
  30 */
  31static void push_buffer_init(struct push_buffer *pb)
  32{
  33        *(u32 *)(pb->mapped + pb->size_bytes) = host1x_opcode_restart(0);
  34}
  35
  36/*
  37 * Increment timedout buffer's syncpt via CPU.
  38 */
  39static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
  40                                u32 syncpt_incrs, u32 syncval, u32 nr_slots)
  41{
  42        struct host1x *host1x = cdma_to_host1x(cdma);
  43        struct push_buffer *pb = &cdma->push_buffer;
  44        u32 i;
  45
  46        for (i = 0; i < syncpt_incrs; i++)
  47                host1x_syncpt_incr(cdma->timeout.syncpt);
  48
  49        /* after CPU incr, ensure shadow is up to date */
  50        host1x_syncpt_load(cdma->timeout.syncpt);
  51
  52        /* NOP all the PB slots */
  53        while (nr_slots--) {
  54                u32 *p = (u32 *)(pb->mapped + getptr);
  55                *(p++) = HOST1X_OPCODE_NOP;
  56                *(p++) = HOST1X_OPCODE_NOP;
  57                dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__,
  58                        &pb->phys, getptr);
  59                getptr = (getptr + 8) & (pb->size_bytes - 1);
  60        }
  61        wmb();
  62}
  63
  64/*
  65 * Start channel DMA
  66 */
  67static void cdma_start(struct host1x_cdma *cdma)
  68{
  69        struct host1x_channel *ch = cdma_to_channel(cdma);
  70
  71        if (cdma->running)
  72                return;
  73
  74        cdma->last_pos = cdma->push_buffer.pos;
  75
  76        host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
  77                         HOST1X_CHANNEL_DMACTRL);
  78
  79        /* set base, put and end pointer */
  80        host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART);
  81        host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
  82        host1x_ch_writel(ch, cdma->push_buffer.phys +
  83                         cdma->push_buffer.size_bytes + 4,
  84                         HOST1X_CHANNEL_DMAEND);
  85
  86        /* reset GET */
  87        host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
  88                         HOST1X_CHANNEL_DMACTRL_DMAGETRST |
  89                         HOST1X_CHANNEL_DMACTRL_DMAINITGET,
  90                         HOST1X_CHANNEL_DMACTRL);
  91
  92        /* start the command DMA */
  93        host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
  94
  95        cdma->running = true;
  96}
  97
  98/*
  99 * Similar to cdma_start(), but rather than starting from an idle
 100 * state (where DMA GET is set to DMA PUT), on a timeout we restore
 101 * DMA GET from an explicit value (so DMA may again be pending).
 102 */
 103static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
 104{
 105        struct host1x *host1x = cdma_to_host1x(cdma);
 106        struct host1x_channel *ch = cdma_to_channel(cdma);
 107
 108        if (cdma->running)
 109                return;
 110
 111        cdma->last_pos = cdma->push_buffer.pos;
 112
 113        host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 114                         HOST1X_CHANNEL_DMACTRL);
 115
 116        /* set base, end pointer (all of memory) */
 117        host1x_ch_writel(ch, cdma->push_buffer.phys, HOST1X_CHANNEL_DMASTART);
 118        host1x_ch_writel(ch, cdma->push_buffer.phys +
 119                         cdma->push_buffer.size_bytes,
 120                         HOST1X_CHANNEL_DMAEND);
 121
 122        /* set GET, by loading the value in PUT (then reset GET) */
 123        host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
 124        host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
 125                         HOST1X_CHANNEL_DMACTRL_DMAGETRST |
 126                         HOST1X_CHANNEL_DMACTRL_DMAINITGET,
 127                         HOST1X_CHANNEL_DMACTRL);
 128
 129        dev_dbg(host1x->dev,
 130                "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
 131                host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 132                host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 133                cdma->last_pos);
 134
 135        /* deassert GET reset and set PUT */
 136        host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 137                         HOST1X_CHANNEL_DMACTRL);
 138        host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
 139
 140        /* start the command DMA */
 141        host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
 142
 143        cdma->running = true;
 144}
 145
 146/*
 147 * Kick channel DMA into action by writing its PUT offset (if it has changed)
 148 */
 149static void cdma_flush(struct host1x_cdma *cdma)
 150{
 151        struct host1x_channel *ch = cdma_to_channel(cdma);
 152
 153        if (cdma->push_buffer.pos != cdma->last_pos) {
 154                host1x_ch_writel(ch, cdma->push_buffer.pos,
 155                                 HOST1X_CHANNEL_DMAPUT);
 156                cdma->last_pos = cdma->push_buffer.pos;
 157        }
 158}
 159
 160static void cdma_stop(struct host1x_cdma *cdma)
 161{
 162        struct host1x_channel *ch = cdma_to_channel(cdma);
 163
 164        mutex_lock(&cdma->lock);
 165        if (cdma->running) {
 166                host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
 167                host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 168                                 HOST1X_CHANNEL_DMACTRL);
 169                cdma->running = false;
 170        }
 171        mutex_unlock(&cdma->lock);
 172}
 173
 174/*
 175 * Stops both channel's command processor and CDMA immediately.
 176 * Also, tears down the channel and resets corresponding module.
 177 */
 178static void cdma_freeze(struct host1x_cdma *cdma)
 179{
 180        struct host1x *host = cdma_to_host1x(cdma);
 181        struct host1x_channel *ch = cdma_to_channel(cdma);
 182        u32 cmdproc_stop;
 183
 184        if (cdma->torndown && !cdma->running) {
 185                dev_warn(host->dev, "Already torn down\n");
 186                return;
 187        }
 188
 189        dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
 190
 191        cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
 192        cmdproc_stop |= BIT(ch->id);
 193        host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 194
 195        dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
 196                __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
 197                host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
 198                cdma->last_pos);
 199
 200        host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 201                         HOST1X_CHANNEL_DMACTRL);
 202
 203        host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
 204
 205        cdma->running = false;
 206        cdma->torndown = true;
 207}
 208
 209static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
 210{
 211        struct host1x *host1x = cdma_to_host1x(cdma);
 212        struct host1x_channel *ch = cdma_to_channel(cdma);
 213        u32 cmdproc_stop;
 214
 215        dev_dbg(host1x->dev,
 216                "resuming channel (id %d, DMAGET restart = 0x%x)\n",
 217                ch->id, getptr);
 218
 219        cmdproc_stop = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP);
 220        cmdproc_stop &= ~(BIT(ch->id));
 221        host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 222
 223        cdma->torndown = false;
 224        cdma_timeout_restart(cdma, getptr);
 225}
 226
 227/*
 228 * If this timeout fires, it indicates the current sync_queue entry has
 229 * exceeded its TTL and the userctx should be timed out and remaining
 230 * submits already issued cleaned up (future submits return an error).
 231 */
 232static void cdma_timeout_handler(struct work_struct *work)
 233{
 234        struct host1x_cdma *cdma;
 235        struct host1x *host1x;
 236        struct host1x_channel *ch;
 237
 238        u32 syncpt_val;
 239
 240        u32 prev_cmdproc, cmdproc_stop;
 241
 242        cdma = container_of(to_delayed_work(work), struct host1x_cdma,
 243                            timeout.wq);
 244        host1x = cdma_to_host1x(cdma);
 245        ch = cdma_to_channel(cdma);
 246
 247        host1x_debug_dump(cdma_to_host1x(cdma));
 248
 249        mutex_lock(&cdma->lock);
 250
 251        if (!cdma->timeout.client) {
 252                dev_dbg(host1x->dev,
 253                        "cdma_timeout: expired, but has no clientid\n");
 254                mutex_unlock(&cdma->lock);
 255                return;
 256        }
 257
 258        /* stop processing to get a clean snapshot */
 259        prev_cmdproc = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP);
 260        cmdproc_stop = prev_cmdproc | BIT(ch->id);
 261        host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 262
 263        dev_dbg(host1x->dev, "cdma_timeout: cmdproc was 0x%x is 0x%x\n",
 264                prev_cmdproc, cmdproc_stop);
 265
 266        syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
 267
 268        /* has buffer actually completed? */
 269        if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
 270                dev_dbg(host1x->dev,
 271                        "cdma_timeout: expired, but buffer had completed\n");
 272                /* restore */
 273                cmdproc_stop = prev_cmdproc & ~(BIT(ch->id));
 274                host1x_sync_writel(host1x, cmdproc_stop,
 275                                   HOST1X_SYNC_CMDPROC_STOP);
 276                mutex_unlock(&cdma->lock);
 277                return;
 278        }
 279
 280        dev_warn(host1x->dev, "%s: timeout: %d (%s), HW thresh %d, done %d\n",
 281                __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
 282                syncpt_val, cdma->timeout.syncpt_val);
 283
 284        /* stop HW, resetting channel/module */
 285        host1x_hw_cdma_freeze(host1x, cdma);
 286
 287        host1x_cdma_update_sync_queue(cdma, ch->dev);
 288        mutex_unlock(&cdma->lock);
 289}
 290
 291/*
 292 * Init timeout resources
 293 */
 294static int cdma_timeout_init(struct host1x_cdma *cdma, u32 syncpt_id)
 295{
 296        INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
 297        cdma->timeout.initialized = true;
 298
 299        return 0;
 300}
 301
 302/*
 303 * Clean up timeout resources
 304 */
 305static void cdma_timeout_destroy(struct host1x_cdma *cdma)
 306{
 307        if (cdma->timeout.initialized)
 308                cancel_delayed_work(&cdma->timeout.wq);
 309        cdma->timeout.initialized = false;
 310}
 311
 312static const struct host1x_cdma_ops host1x_cdma_ops = {
 313        .start = cdma_start,
 314        .stop = cdma_stop,
 315        .flush = cdma_flush,
 316
 317        .timeout_init = cdma_timeout_init,
 318        .timeout_destroy = cdma_timeout_destroy,
 319        .freeze = cdma_freeze,
 320        .resume = cdma_resume,
 321        .timeout_cpu_incr = cdma_timeout_cpu_incr,
 322};
 323
 324static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
 325        .init = push_buffer_init,
 326};
 327