linux/drivers/gpu/host1x/cdma.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Tegra host1x Command DMA
   4 *
   5 * Copyright (c) 2010-2013, NVIDIA Corporation.
   6 */
   7
   8
   9#include <asm/cacheflush.h>
  10#include <linux/device.h>
  11#include <linux/dma-mapping.h>
  12#include <linux/host1x.h>
  13#include <linux/interrupt.h>
  14#include <linux/kernel.h>
  15#include <linux/kfifo.h>
  16#include <linux/slab.h>
  17#include <trace/events/host1x.h>
  18
  19#include "cdma.h"
  20#include "channel.h"
  21#include "dev.h"
  22#include "debug.h"
  23#include "job.h"
  24
  25/*
  26 * push_buffer
  27 *
  28 * The push buffer is a circular array of words to be fetched by command DMA.
  29 * Note that it works slightly differently to the sync queue; fence == pos
  30 * means that the push buffer is full, not empty.
  31 */
  32
  33/*
  34 * Typically the commands written into the push buffer are a pair of words. We
  35 * use slots to represent each of these pairs and to simplify things. Note the
  36 * strange number of slots allocated here. 512 slots will fit exactly within a
  37 * single memory page. We also need one additional word at the end of the push
  38 * buffer for the RESTART opcode that will instruct the CDMA to jump back to
  39 * the beginning of the push buffer. With 512 slots, this means that we'll use
  40 * 2 memory pages and waste 4092 bytes of the second page that will never be
  41 * used.
  42 */
  43#define HOST1X_PUSHBUFFER_SLOTS 511
  44
  45/*
  46 * Clean up push buffer resources
  47 */
  48static void host1x_pushbuffer_destroy(struct push_buffer *pb)
  49{
  50        struct host1x_cdma *cdma = pb_to_cdma(pb);
  51        struct host1x *host1x = cdma_to_host1x(cdma);
  52
  53        if (!pb->mapped)
  54                return;
  55
  56        if (host1x->domain) {
  57                iommu_unmap(host1x->domain, pb->dma, pb->alloc_size);
  58                free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma));
  59        }
  60
  61        dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);
  62
  63        pb->mapped = NULL;
  64        pb->phys = 0;
  65}
  66
  67/*
  68 * Init push buffer resources
  69 */
  70static int host1x_pushbuffer_init(struct push_buffer *pb)
  71{
  72        struct host1x_cdma *cdma = pb_to_cdma(pb);
  73        struct host1x *host1x = cdma_to_host1x(cdma);
  74        struct iova *alloc;
  75        u32 size;
  76        int err;
  77
  78        pb->mapped = NULL;
  79        pb->phys = 0;
  80        pb->size = HOST1X_PUSHBUFFER_SLOTS * 8;
  81
  82        size = pb->size + 4;
  83
  84        /* initialize buffer pointers */
  85        pb->fence = pb->size - 8;
  86        pb->pos = 0;
  87
  88        if (host1x->domain) {
  89                unsigned long shift;
  90
  91                size = iova_align(&host1x->iova, size);
  92
  93                pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
  94                                          GFP_KERNEL);
  95                if (!pb->mapped)
  96                        return -ENOMEM;
  97
  98                shift = iova_shift(&host1x->iova);
  99                alloc = alloc_iova(&host1x->iova, size >> shift,
 100                                   host1x->iova_end >> shift, true);
 101                if (!alloc) {
 102                        err = -ENOMEM;
 103                        goto iommu_free_mem;
 104                }
 105
 106                pb->dma = iova_dma_addr(&host1x->iova, alloc);
 107                err = iommu_map(host1x->domain, pb->dma, pb->phys, size,
 108                                IOMMU_READ);
 109                if (err)
 110                        goto iommu_free_iova;
 111        } else {
 112                pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
 113                                          GFP_KERNEL);
 114                if (!pb->mapped)
 115                        return -ENOMEM;
 116
 117                pb->dma = pb->phys;
 118        }
 119
 120        pb->alloc_size = size;
 121
 122        host1x_hw_pushbuffer_init(host1x, pb);
 123
 124        return 0;
 125
 126iommu_free_iova:
 127        __free_iova(&host1x->iova, alloc);
 128iommu_free_mem:
 129        dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);
 130
 131        return err;
 132}
 133
 134/*
 135 * Push two words to the push buffer
 136 * Caller must ensure push buffer is not full
 137 */
 138static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
 139{
 140        u32 *p = (u32 *)((void *)pb->mapped + pb->pos);
 141
 142        WARN_ON(pb->pos == pb->fence);
 143        *(p++) = op1;
 144        *(p++) = op2;
 145        pb->pos += 8;
 146
 147        if (pb->pos >= pb->size)
 148                pb->pos -= pb->size;
 149}
 150
 151/*
 152 * Pop a number of two word slots from the push buffer
 153 * Caller must ensure push buffer is not empty
 154 */
 155static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
 156{
 157        /* Advance the next write position */
 158        pb->fence += slots * 8;
 159
 160        if (pb->fence >= pb->size)
 161                pb->fence -= pb->size;
 162}
 163
 164/*
 165 * Return the number of two word slots free in the push buffer
 166 */
 167static u32 host1x_pushbuffer_space(struct push_buffer *pb)
 168{
 169        unsigned int fence = pb->fence;
 170
 171        if (pb->fence < pb->pos)
 172                fence += pb->size;
 173
 174        return (fence - pb->pos) / 8;
 175}
 176
 177/*
 178 * Sleep (if necessary) until the requested event happens
 179 *   - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
 180 *     - Returns 1
 181 *   - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
 182 *     - Return the amount of space (> 0)
 183 * Must be called with the cdma lock held.
 184 */
 185unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
 186                                     enum cdma_event event)
 187{
 188        for (;;) {
 189                struct push_buffer *pb = &cdma->push_buffer;
 190                unsigned int space;
 191
 192                switch (event) {
 193                case CDMA_EVENT_SYNC_QUEUE_EMPTY:
 194                        space = list_empty(&cdma->sync_queue) ? 1 : 0;
 195                        break;
 196
 197                case CDMA_EVENT_PUSH_BUFFER_SPACE:
 198                        space = host1x_pushbuffer_space(pb);
 199                        break;
 200
 201                default:
 202                        WARN_ON(1);
 203                        return -EINVAL;
 204                }
 205
 206                if (space)
 207                        return space;
 208
 209                trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
 210                                       event);
 211
 212                /* If somebody has managed to already start waiting, yield */
 213                if (cdma->event != CDMA_EVENT_NONE) {
 214                        mutex_unlock(&cdma->lock);
 215                        schedule();
 216                        mutex_lock(&cdma->lock);
 217                        continue;
 218                }
 219
 220                cdma->event = event;
 221
 222                mutex_unlock(&cdma->lock);
 223                wait_for_completion(&cdma->complete);
 224                mutex_lock(&cdma->lock);
 225        }
 226
 227        return 0;
 228}
 229
 230/*
 231 * Sleep (if necessary) until the push buffer has enough free space.
 232 *
 233 * Must be called with the cdma lock held.
 234 */
 235static int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x,
 236                                             struct host1x_cdma *cdma,
 237                                             unsigned int needed)
 238{
 239        while (true) {
 240                struct push_buffer *pb = &cdma->push_buffer;
 241                unsigned int space;
 242
 243                space = host1x_pushbuffer_space(pb);
 244                if (space >= needed)
 245                        break;
 246
 247                trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
 248                                       CDMA_EVENT_PUSH_BUFFER_SPACE);
 249
 250                host1x_hw_cdma_flush(host1x, cdma);
 251
 252                /* If somebody has managed to already start waiting, yield */
 253                if (cdma->event != CDMA_EVENT_NONE) {
 254                        mutex_unlock(&cdma->lock);
 255                        schedule();
 256                        mutex_lock(&cdma->lock);
 257                        continue;
 258                }
 259
 260                cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE;
 261
 262                mutex_unlock(&cdma->lock);
 263                wait_for_completion(&cdma->complete);
 264                mutex_lock(&cdma->lock);
 265        }
 266
 267        return 0;
 268}
 269/*
 270 * Start timer that tracks the time spent by the job.
 271 * Must be called with the cdma lock held.
 272 */
 273static void cdma_start_timer_locked(struct host1x_cdma *cdma,
 274                                    struct host1x_job *job)
 275{
 276        struct host1x *host = cdma_to_host1x(cdma);
 277
 278        if (cdma->timeout.client) {
 279                /* timer already started */
 280                return;
 281        }
 282
 283        cdma->timeout.client = job->client;
 284        cdma->timeout.syncpt = host1x_syncpt_get(host, job->syncpt_id);
 285        cdma->timeout.syncpt_val = job->syncpt_end;
 286        cdma->timeout.start_ktime = ktime_get();
 287
 288        schedule_delayed_work(&cdma->timeout.wq,
 289                              msecs_to_jiffies(job->timeout));
 290}
 291
 292/*
 293 * Stop timer when a buffer submission completes.
 294 * Must be called with the cdma lock held.
 295 */
 296static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
 297{
 298        cancel_delayed_work(&cdma->timeout.wq);
 299        cdma->timeout.client = NULL;
 300}
 301
 302/*
 303 * For all sync queue entries that have already finished according to the
 304 * current sync point registers:
 305 *  - unpin & unref their mems
 306 *  - pop their push buffer slots
 307 *  - remove them from the sync queue
 308 * This is normally called from the host code's worker thread, but can be
 309 * called manually if necessary.
 310 * Must be called with the cdma lock held.
 311 */
 312static void update_cdma_locked(struct host1x_cdma *cdma)
 313{
 314        bool signal = false;
 315        struct host1x *host1x = cdma_to_host1x(cdma);
 316        struct host1x_job *job, *n;
 317
 318        /* If CDMA is stopped, queue is cleared and we can return */
 319        if (!cdma->running)
 320                return;
 321
 322        /*
 323         * Walk the sync queue, reading the sync point registers as necessary,
 324         * to consume as many sync queue entries as possible without blocking
 325         */
 326        list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
 327                struct host1x_syncpt *sp =
 328                        host1x_syncpt_get(host1x, job->syncpt_id);
 329
 330                /* Check whether this syncpt has completed, and bail if not */
 331                if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) {
 332                        /* Start timer on next pending syncpt */
 333                        if (job->timeout)
 334                                cdma_start_timer_locked(cdma, job);
 335
 336                        break;
 337                }
 338
 339                /* Cancel timeout, when a buffer completes */
 340                if (cdma->timeout.client)
 341                        stop_cdma_timer_locked(cdma);
 342
 343                /* Unpin the memory */
 344                host1x_job_unpin(job);
 345
 346                /* Pop push buffer slots */
 347                if (job->num_slots) {
 348                        struct push_buffer *pb = &cdma->push_buffer;
 349
 350                        host1x_pushbuffer_pop(pb, job->num_slots);
 351
 352                        if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
 353                                signal = true;
 354                }
 355
 356                list_del(&job->list);
 357                host1x_job_put(job);
 358        }
 359
 360        if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
 361            list_empty(&cdma->sync_queue))
 362                signal = true;
 363
 364        if (signal) {
 365                cdma->event = CDMA_EVENT_NONE;
 366                complete(&cdma->complete);
 367        }
 368}
 369
 370void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
 371                                   struct device *dev)
 372{
 373        struct host1x *host1x = cdma_to_host1x(cdma);
 374        u32 restart_addr, syncpt_incrs, syncpt_val;
 375        struct host1x_job *job, *next_job = NULL;
 376
 377        syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
 378
 379        dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
 380                __func__, syncpt_val);
 381
 382        /*
 383         * Move the sync_queue read pointer to the first entry that hasn't
 384         * completed based on the current HW syncpt value. It's likely there
 385         * won't be any (i.e. we're still at the head), but covers the case
 386         * where a syncpt incr happens just prior/during the teardown.
 387         */
 388
 389        dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
 390                __func__);
 391
 392        list_for_each_entry(job, &cdma->sync_queue, list) {
 393                if (syncpt_val < job->syncpt_end) {
 394
 395                        if (!list_is_last(&job->list, &cdma->sync_queue))
 396                                next_job = list_next_entry(job, list);
 397
 398                        goto syncpt_incr;
 399                }
 400
 401                host1x_job_dump(dev, job);
 402        }
 403
 404        /* all jobs have been completed */
 405        job = NULL;
 406
 407syncpt_incr:
 408
 409        /*
 410         * Increment with CPU the remaining syncpts of a partially executed job.
 411         *
 412         * CDMA will continue execution starting with the next job or will get
 413         * into idle state.
 414         */
 415        if (next_job)
 416                restart_addr = next_job->first_get;
 417        else
 418                restart_addr = cdma->last_pos;
 419
 420        /* do CPU increments for the remaining syncpts */
 421        if (job) {
 422                dev_dbg(dev, "%s: perform CPU incr on pending buffers\n",
 423                        __func__);
 424
 425                /* won't need a timeout when replayed */
 426                job->timeout = 0;
 427
 428                syncpt_incrs = job->syncpt_end - syncpt_val;
 429                dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);
 430
 431                host1x_job_dump(dev, job);
 432
 433                /* safe to use CPU to incr syncpts */
 434                host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
 435                                                syncpt_incrs, job->syncpt_end,
 436                                                job->num_slots);
 437
 438                dev_dbg(dev, "%s: finished sync_queue modification\n",
 439                        __func__);
 440        }
 441
 442        /* roll back DMAGET and start up channel again */
 443        host1x_hw_cdma_resume(host1x, cdma, restart_addr);
 444}
 445
 446/*
 447 * Create a cdma
 448 */
 449int host1x_cdma_init(struct host1x_cdma *cdma)
 450{
 451        int err;
 452
 453        mutex_init(&cdma->lock);
 454        init_completion(&cdma->complete);
 455
 456        INIT_LIST_HEAD(&cdma->sync_queue);
 457
 458        cdma->event = CDMA_EVENT_NONE;
 459        cdma->running = false;
 460        cdma->torndown = false;
 461
 462        err = host1x_pushbuffer_init(&cdma->push_buffer);
 463        if (err)
 464                return err;
 465
 466        return 0;
 467}
 468
 469/*
 470 * Destroy a cdma
 471 */
 472int host1x_cdma_deinit(struct host1x_cdma *cdma)
 473{
 474        struct push_buffer *pb = &cdma->push_buffer;
 475        struct host1x *host1x = cdma_to_host1x(cdma);
 476
 477        if (cdma->running) {
 478                pr_warn("%s: CDMA still running\n", __func__);
 479                return -EBUSY;
 480        }
 481
 482        host1x_pushbuffer_destroy(pb);
 483        host1x_hw_cdma_timeout_destroy(host1x, cdma);
 484
 485        return 0;
 486}
 487
 488/*
 489 * Begin a cdma submit
 490 */
 491int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
 492{
 493        struct host1x *host1x = cdma_to_host1x(cdma);
 494
 495        mutex_lock(&cdma->lock);
 496
 497        if (job->timeout) {
 498                /* init state on first submit with timeout value */
 499                if (!cdma->timeout.initialized) {
 500                        int err;
 501
 502                        err = host1x_hw_cdma_timeout_init(host1x, cdma,
 503                                                          job->syncpt_id);
 504                        if (err) {
 505                                mutex_unlock(&cdma->lock);
 506                                return err;
 507                        }
 508                }
 509        }
 510
 511        if (!cdma->running)
 512                host1x_hw_cdma_start(host1x, cdma);
 513
 514        cdma->slots_free = 0;
 515        cdma->slots_used = 0;
 516        cdma->first_get = cdma->push_buffer.pos;
 517
 518        trace_host1x_cdma_begin(dev_name(job->channel->dev));
 519        return 0;
 520}
 521
 522/*
 523 * Push two words into a push buffer slot
 524 * Blocks as necessary if the push buffer is full.
 525 */
 526void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
 527{
 528        struct host1x *host1x = cdma_to_host1x(cdma);
 529        struct push_buffer *pb = &cdma->push_buffer;
 530        u32 slots_free = cdma->slots_free;
 531
 532        if (host1x_debug_trace_cmdbuf)
 533                trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
 534                                       op1, op2);
 535
 536        if (slots_free == 0) {
 537                host1x_hw_cdma_flush(host1x, cdma);
 538                slots_free = host1x_cdma_wait_locked(cdma,
 539                                                CDMA_EVENT_PUSH_BUFFER_SPACE);
 540        }
 541
 542        cdma->slots_free = slots_free - 1;
 543        cdma->slots_used++;
 544        host1x_pushbuffer_push(pb, op1, op2);
 545}
 546
 547/*
 548 * Push four words into two consecutive push buffer slots. Note that extra
 549 * care needs to be taken not to split the two slots across the end of the
 550 * push buffer. Otherwise the RESTART opcode at the end of the push buffer
 551 * that ensures processing will restart at the beginning will break up the
 552 * four words.
 553 *
 554 * Blocks as necessary if the push buffer is full.
 555 */
 556void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2,
 557                           u32 op3, u32 op4)
 558{
 559        struct host1x_channel *channel = cdma_to_channel(cdma);
 560        struct host1x *host1x = cdma_to_host1x(cdma);
 561        struct push_buffer *pb = &cdma->push_buffer;
 562        unsigned int needed = 2, extra = 0, i;
 563        unsigned int space = cdma->slots_free;
 564
 565        if (host1x_debug_trace_cmdbuf)
 566                trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2,
 567                                            op3, op4);
 568
 569        /* compute number of extra slots needed for padding */
 570        if (pb->pos + 16 > pb->size) {
 571                extra = (pb->size - pb->pos) / 8;
 572                needed += extra;
 573        }
 574
 575        host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed);
 576        space = host1x_pushbuffer_space(pb);
 577
 578        cdma->slots_free = space - needed;
 579        cdma->slots_used += needed;
 580
 581        /*
 582         * Note that we rely on the fact that this is only used to submit wide
 583         * gather opcodes, which consist of 3 words, and they are padded with
 584         * a NOP to avoid having to deal with fractional slots (a slot always
 585         * represents 2 words). The fourth opcode passed to this function will
 586         * therefore always be a NOP.
 587         *
 588         * This works around a slight ambiguity when it comes to opcodes. For
 589         * all current host1x incarnations the NOP opcode uses the exact same
 590         * encoding (0x20000000), so we could hard-code the value here, but a
 591         * new incarnation may change it and break that assumption.
 592         */
 593        for (i = 0; i < extra; i++)
 594                host1x_pushbuffer_push(pb, op4, op4);
 595
 596        host1x_pushbuffer_push(pb, op1, op2);
 597        host1x_pushbuffer_push(pb, op3, op4);
 598}
 599
 600/*
 601 * End a cdma submit
 602 * Kick off DMA, add job to the sync queue, and a number of slots to be freed
 603 * from the pushbuffer. The handles for a submit must all be pinned at the same
 604 * time, but they can be unpinned in smaller chunks.
 605 */
 606void host1x_cdma_end(struct host1x_cdma *cdma,
 607                     struct host1x_job *job)
 608{
 609        struct host1x *host1x = cdma_to_host1x(cdma);
 610        bool idle = list_empty(&cdma->sync_queue);
 611
 612        host1x_hw_cdma_flush(host1x, cdma);
 613
 614        job->first_get = cdma->first_get;
 615        job->num_slots = cdma->slots_used;
 616        host1x_job_get(job);
 617        list_add_tail(&job->list, &cdma->sync_queue);
 618
 619        /* start timer on idle -> active transitions */
 620        if (job->timeout && idle)
 621                cdma_start_timer_locked(cdma, job);
 622
 623        trace_host1x_cdma_end(dev_name(job->channel->dev));
 624        mutex_unlock(&cdma->lock);
 625}
 626
 627/*
 628 * Update cdma state according to current sync point values
 629 */
 630void host1x_cdma_update(struct host1x_cdma *cdma)
 631{
 632        mutex_lock(&cdma->lock);
 633        update_cdma_locked(cdma);
 634        mutex_unlock(&cdma->lock);
 635}
 636