linux/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
<<
>>
Prefs
   1/*
   2 * SPDX-License-Identifier: MIT
   3 *
   4 * Copyright © 2008,2010 Intel Corporation
   5 */
   6
   7#include <linux/intel-iommu.h>
   8#include <linux/dma-resv.h>
   9#include <linux/sync_file.h>
  10#include <linux/uaccess.h>
  11
  12#include <drm/drm_syncobj.h>
  13#include <drm/i915_drm.h>
  14
  15#include "display/intel_frontbuffer.h"
  16
  17#include "gem/i915_gem_ioctls.h"
  18#include "gt/intel_context.h"
  19#include "gt/intel_engine_pool.h"
  20#include "gt/intel_gt.h"
  21#include "gt/intel_gt_pm.h"
  22
  23#include "i915_drv.h"
  24#include "i915_gem_clflush.h"
  25#include "i915_gem_context.h"
  26#include "i915_gem_ioctls.h"
  27#include "i915_trace.h"
  28
  29enum {
  30        FORCE_CPU_RELOC = 1,
  31        FORCE_GTT_RELOC,
  32        FORCE_GPU_RELOC,
  33#define DBG_FORCE_RELOC 0 /* choose one of the above! */
  34};
  35
  36#define __EXEC_OBJECT_HAS_REF           BIT(31)
  37#define __EXEC_OBJECT_HAS_PIN           BIT(30)
  38#define __EXEC_OBJECT_HAS_FENCE         BIT(29)
  39#define __EXEC_OBJECT_NEEDS_MAP         BIT(28)
  40#define __EXEC_OBJECT_NEEDS_BIAS        BIT(27)
  41#define __EXEC_OBJECT_INTERNAL_FLAGS    (~0u << 27) /* all of the above */
  42#define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
  43
  44#define __EXEC_HAS_RELOC        BIT(31)
  45#define __EXEC_VALIDATED        BIT(30)
  46#define __EXEC_INTERNAL_FLAGS   (~0u << 30)
  47#define UPDATE                  PIN_OFFSET_FIXED
  48
  49#define BATCH_OFFSET_BIAS (256*1024)
  50
  51#define __I915_EXEC_ILLEGAL_FLAGS \
  52        (__I915_EXEC_UNKNOWN_FLAGS | \
  53         I915_EXEC_CONSTANTS_MASK  | \
  54         I915_EXEC_RESOURCE_STREAMER)
  55
  56/* Catch emission of unexpected errors for CI! */
  57#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
  58#undef EINVAL
  59#define EINVAL ({ \
  60        DRM_DEBUG_DRIVER("EINVAL at %s:%d\n", __func__, __LINE__); \
  61        22; \
  62})
  63#endif
  64
  65/**
  66 * DOC: User command execution
  67 *
  68 * Userspace submits commands to be executed on the GPU as an instruction
  69 * stream within a GEM object we call a batchbuffer. This instructions may
  70 * refer to other GEM objects containing auxiliary state such as kernels,
  71 * samplers, render targets and even secondary batchbuffers. Userspace does
  72 * not know where in the GPU memory these objects reside and so before the
  73 * batchbuffer is passed to the GPU for execution, those addresses in the
  74 * batchbuffer and auxiliary objects are updated. This is known as relocation,
  75 * or patching. To try and avoid having to relocate each object on the next
  76 * execution, userspace is told the location of those objects in this pass,
  77 * but this remains just a hint as the kernel may choose a new location for
  78 * any object in the future.
  79 *
  80 * At the level of talking to the hardware, submitting a batchbuffer for the
  81 * GPU to execute is to add content to a buffer from which the HW
  82 * command streamer is reading.
  83 *
  84 * 1. Add a command to load the HW context. For Logical Ring Contexts, i.e.
  85 *    Execlists, this command is not placed on the same buffer as the
  86 *    remaining items.
  87 *
  88 * 2. Add a command to invalidate caches to the buffer.
  89 *
  90 * 3. Add a batchbuffer start command to the buffer; the start command is
  91 *    essentially a token together with the GPU address of the batchbuffer
  92 *    to be executed.
  93 *
  94 * 4. Add a pipeline flush to the buffer.
  95 *
  96 * 5. Add a memory write command to the buffer to record when the GPU
  97 *    is done executing the batchbuffer. The memory write writes the
  98 *    global sequence number of the request, ``i915_request::global_seqno``;
  99 *    the i915 driver uses the current value in the register to determine
 100 *    if the GPU has completed the batchbuffer.
 101 *
 102 * 6. Add a user interrupt command to the buffer. This command instructs
 103 *    the GPU to issue an interrupt when the command, pipeline flush and
 104 *    memory write are completed.
 105 *
 106 * 7. Inform the hardware of the additional commands added to the buffer
 107 *    (by updating the tail pointer).
 108 *
 109 * Processing an execbuf ioctl is conceptually split up into a few phases.
 110 *
 111 * 1. Validation - Ensure all the pointers, handles and flags are valid.
 112 * 2. Reservation - Assign GPU address space for every object
 113 * 3. Relocation - Update any addresses to point to the final locations
 114 * 4. Serialisation - Order the request with respect to its dependencies
 115 * 5. Construction - Construct a request to execute the batchbuffer
 116 * 6. Submission (at some point in the future execution)
 117 *
 118 * Reserving resources for the execbuf is the most complicated phase. We
 119 * neither want to have to migrate the object in the address space, nor do
 120 * we want to have to update any relocations pointing to this object. Ideally,
 121 * we want to leave the object where it is and for all the existing relocations
 122 * to match. If the object is given a new address, or if userspace thinks the
 123 * object is elsewhere, we have to parse all the relocation entries and update
 124 * the addresses. Userspace can set the I915_EXEC_NORELOC flag to hint that
 125 * all the target addresses in all of its objects match the value in the
 126 * relocation entries and that they all match the presumed offsets given by the
 127 * list of execbuffer objects. Using this knowledge, we know that if we haven't
 128 * moved any buffers, all the relocation entries are valid and we can skip
 129 * the update. (If userspace is wrong, the likely outcome is an impromptu GPU
 130 * hang.) The requirement for using I915_EXEC_NO_RELOC are:
 131 *
 132 *      The addresses written in the objects must match the corresponding
 133 *      reloc.presumed_offset which in turn must match the corresponding
 134 *      execobject.offset.
 135 *
 136 *      Any render targets written to in the batch must be flagged with
 137 *      EXEC_OBJECT_WRITE.
 138 *
 139 *      To avoid stalling, execobject.offset should match the current
 140 *      address of that object within the active context.
 141 *
 142 * The reservation is done is multiple phases. First we try and keep any
 143 * object already bound in its current location - so as long as meets the
 144 * constraints imposed by the new execbuffer. Any object left unbound after the
 145 * first pass is then fitted into any available idle space. If an object does
 146 * not fit, all objects are removed from the reservation and the process rerun
 147 * after sorting the objects into a priority order (more difficult to fit
 148 * objects are tried first). Failing that, the entire VM is cleared and we try
 149 * to fit the execbuf once last time before concluding that it simply will not
 150 * fit.
 151 *
 152 * A small complication to all of this is that we allow userspace not only to
 153 * specify an alignment and a size for the object in the address space, but
 154 * we also allow userspace to specify the exact offset. This objects are
 155 * simpler to place (the location is known a priori) all we have to do is make
 156 * sure the space is available.
 157 *
 158 * Once all the objects are in place, patching up the buried pointers to point
 159 * to the final locations is a fairly simple job of walking over the relocation
 160 * entry arrays, looking up the right address and rewriting the value into
 161 * the object. Simple! ... The relocation entries are stored in user memory
 162 * and so to access them we have to copy them into a local buffer. That copy
 163 * has to avoid taking any pagefaults as they may lead back to a GEM object
 164 * requiring the struct_mutex (i.e. recursive deadlock). So once again we split
 165 * the relocation into multiple passes. First we try to do everything within an
 166 * atomic context (avoid the pagefaults) which requires that we never wait. If
 167 * we detect that we may wait, or if we need to fault, then we have to fallback
 168 * to a slower path. The slowpath has to drop the mutex. (Can you hear alarm
 169 * bells yet?) Dropping the mutex means that we lose all the state we have
 170 * built up so far for the execbuf and we must reset any global data. However,
 171 * we do leave the objects pinned in their final locations - which is a
 172 * potential issue for concurrent execbufs. Once we have left the mutex, we can
 173 * allocate and copy all the relocation entries into a large array at our
 174 * leisure, reacquire the mutex, reclaim all the objects and other state and
 175 * then proceed to update any incorrect addresses with the objects.
 176 *
 177 * As we process the relocation entries, we maintain a record of whether the
 178 * object is being written to. Using NORELOC, we expect userspace to provide
 179 * this information instead. We also check whether we can skip the relocation
 180 * by comparing the expected value inside the relocation entry with the target's
 181 * final address. If they differ, we have to map the current object and rewrite
 182 * the 4 or 8 byte pointer within.
 183 *
 184 * Serialising an execbuf is quite simple according to the rules of the GEM
 185 * ABI. Execution within each context is ordered by the order of submission.
 186 * Writes to any GEM object are in order of submission and are exclusive. Reads
 187 * from a GEM object are unordered with respect to other reads, but ordered by
 188 * writes. A write submitted after a read cannot occur before the read, and
 189 * similarly any read submitted after a write cannot occur before the write.
 190 * Writes are ordered between engines such that only one write occurs at any
 191 * time (completing any reads beforehand) - using semaphores where available
 192 * and CPU serialisation otherwise. Other GEM access obey the same rules, any
 193 * write (either via mmaps using set-domain, or via pwrite) must flush all GPU
 194 * reads before starting, and any read (either using set-domain or pread) must
 195 * flush all GPU writes before starting. (Note we only employ a barrier before,
 196 * we currently rely on userspace not concurrently starting a new execution
 197 * whilst reading or writing to an object. This may be an advantage or not
 198 * depending on how much you trust userspace not to shoot themselves in the
 199 * foot.) Serialisation may just result in the request being inserted into
 200 * a DAG awaiting its turn, but most simple is to wait on the CPU until
 201 * all dependencies are resolved.
 202 *
 203 * After all of that, is just a matter of closing the request and handing it to
 204 * the hardware (well, leaving it in a queue to be executed). However, we also
 205 * offer the ability for batchbuffers to be run with elevated privileges so
 206 * that they access otherwise hidden registers. (Used to adjust L3 cache etc.)
 207 * Before any batch is given extra privileges we first must check that it
 208 * contains no nefarious instructions, we check that each instruction is from
 209 * our whitelist and all registers are also from an allowed list. We first
 210 * copy the user's batchbuffer to a shadow (so that the user doesn't have
 211 * access to it, either by the CPU or GPU as we scan it) and then parse each
 212 * instruction. If everything is ok, we set a flag telling the hardware to run
 213 * the batchbuffer in trusted mode, otherwise the ioctl is rejected.
 214 */
 215
 216struct i915_execbuffer {
 217        struct drm_i915_private *i915; /** i915 backpointer */
 218        struct drm_file *file; /** per-file lookup tables and limits */
 219        struct drm_i915_gem_execbuffer2 *args; /** ioctl parameters */
 220        struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */
 221        struct i915_vma **vma;
 222        unsigned int *flags;
 223
 224        struct intel_engine_cs *engine; /** engine to queue the request to */
 225        struct intel_context *context; /* logical state for the request */
 226        struct i915_gem_context *gem_context; /** caller's context */
 227
 228        struct i915_request *request; /** our request to build */
 229        struct i915_vma *batch; /** identity of the batch obj/vma */
 230
 231        /** actual size of execobj[] as we may extend it for the cmdparser */
 232        unsigned int buffer_count;
 233
 234        /** list of vma not yet bound during reservation phase */
 235        struct list_head unbound;
 236
 237        /** list of vma that have execobj.relocation_count */
 238        struct list_head relocs;
 239
 240        /**
 241         * Track the most recently used object for relocations, as we
 242         * frequently have to perform multiple relocations within the same
 243         * obj/page
 244         */
 245        struct reloc_cache {
 246                struct drm_mm_node node; /** temporary GTT binding */
 247                unsigned long vaddr; /** Current kmap address */
 248                unsigned long page; /** Currently mapped page index */
 249                unsigned int gen; /** Cached value of INTEL_GEN */
 250                bool use_64bit_reloc : 1;
 251                bool has_llc : 1;
 252                bool has_fence : 1;
 253                bool needs_unfenced : 1;
 254
 255                struct i915_request *rq;
 256                u32 *rq_cmd;
 257                unsigned int rq_size;
 258        } reloc_cache;
 259
 260        u64 invalid_flags; /** Set of execobj.flags that are invalid */
 261        u32 context_flags; /** Set of execobj.flags to insert from the ctx */
 262
 263        u32 batch_start_offset; /** Location within object of batch */
 264        u32 batch_len; /** Length of batch within object */
 265        u32 batch_flags; /** Flags composed for emit_bb_start() */
 266
 267        /**
 268         * Indicate either the size of the hastable used to resolve
 269         * relocation handles, or if negative that we are using a direct
 270         * index into the execobj[].
 271         */
 272        int lut_size;
 273        struct hlist_head *buckets; /** ht for relocation handles */
 274};
 275
 276#define exec_entry(EB, VMA) (&(EB)->exec[(VMA)->exec_flags - (EB)->flags])
 277
 278/*
 279 * Used to convert any address to canonical form.
 280 * Starting from gen8, some commands (e.g. STATE_BASE_ADDRESS,
 281 * MI_LOAD_REGISTER_MEM and others, see Broadwell PRM Vol2a) require the
 282 * addresses to be in a canonical form:
 283 * "GraphicsAddress[63:48] are ignored by the HW and assumed to be in correct
 284 * canonical form [63:48] == [47]."
 285 */
 286#define GEN8_HIGH_ADDRESS_BIT 47
 287static inline u64 gen8_canonical_addr(u64 address)
 288{
 289        return sign_extend64(address, GEN8_HIGH_ADDRESS_BIT);
 290}
 291
 292static inline u64 gen8_noncanonical_addr(u64 address)
 293{
 294        return address & GENMASK_ULL(GEN8_HIGH_ADDRESS_BIT, 0);
 295}
 296
 297static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb)
 298{
 299        return intel_engine_requires_cmd_parser(eb->engine) ||
 300                (intel_engine_using_cmd_parser(eb->engine) &&
 301                 eb->args->batch_len);
 302}
 303
 304static int eb_create(struct i915_execbuffer *eb)
 305{
 306        if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) {
 307                unsigned int size = 1 + ilog2(eb->buffer_count);
 308
 309                /*
 310                 * Without a 1:1 association between relocation handles and
 311                 * the execobject[] index, we instead create a hashtable.
 312                 * We size it dynamically based on available memory, starting
 313                 * first with 1:1 assocative hash and scaling back until
 314                 * the allocation succeeds.
 315                 *
 316                 * Later on we use a positive lut_size to indicate we are
 317                 * using this hashtable, and a negative value to indicate a
 318                 * direct lookup.
 319                 */
 320                do {
 321                        gfp_t flags;
 322
 323                        /* While we can still reduce the allocation size, don't
 324                         * raise a warning and allow the allocation to fail.
 325                         * On the last pass though, we want to try as hard
 326                         * as possible to perform the allocation and warn
 327                         * if it fails.
 328                         */
 329                        flags = GFP_KERNEL;
 330                        if (size > 1)
 331                                flags |= __GFP_NORETRY | __GFP_NOWARN;
 332
 333                        eb->buckets = kzalloc(sizeof(struct hlist_head) << size,
 334                                              flags);
 335                        if (eb->buckets)
 336                                break;
 337                } while (--size);
 338
 339                if (unlikely(!size))
 340                        return -ENOMEM;
 341
 342                eb->lut_size = size;
 343        } else {
 344                eb->lut_size = -eb->buffer_count;
 345        }
 346
 347        return 0;
 348}
 349
 350static bool
 351eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry,
 352                 const struct i915_vma *vma,
 353                 unsigned int flags)
 354{
 355        if (vma->node.size < entry->pad_to_size)
 356                return true;
 357
 358        if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment))
 359                return true;
 360
 361        if (flags & EXEC_OBJECT_PINNED &&
 362            vma->node.start != entry->offset)
 363                return true;
 364
 365        if (flags & __EXEC_OBJECT_NEEDS_BIAS &&
 366            vma->node.start < BATCH_OFFSET_BIAS)
 367                return true;
 368
 369        if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) &&
 370            (vma->node.start + vma->node.size - 1) >> 32)
 371                return true;
 372
 373        if (flags & __EXEC_OBJECT_NEEDS_MAP &&
 374            !i915_vma_is_map_and_fenceable(vma))
 375                return true;
 376
 377        return false;
 378}
 379
 380static inline bool
 381eb_pin_vma(struct i915_execbuffer *eb,
 382           const struct drm_i915_gem_exec_object2 *entry,
 383           struct i915_vma *vma)
 384{
 385        unsigned int exec_flags = *vma->exec_flags;
 386        u64 pin_flags;
 387
 388        if (vma->node.size)
 389                pin_flags = vma->node.start;
 390        else
 391                pin_flags = entry->offset & PIN_OFFSET_MASK;
 392
 393        pin_flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED;
 394        if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_GTT))
 395                pin_flags |= PIN_GLOBAL;
 396
 397        if (unlikely(i915_vma_pin(vma, 0, 0, pin_flags)))
 398                return false;
 399
 400        if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
 401                if (unlikely(i915_vma_pin_fence(vma))) {
 402                        i915_vma_unpin(vma);
 403                        return false;
 404                }
 405
 406                if (vma->fence)
 407                        exec_flags |= __EXEC_OBJECT_HAS_FENCE;
 408        }
 409
 410        *vma->exec_flags = exec_flags | __EXEC_OBJECT_HAS_PIN;
 411        return !eb_vma_misplaced(entry, vma, exec_flags);
 412}
 413
 414static inline void __eb_unreserve_vma(struct i915_vma *vma, unsigned int flags)
 415{
 416        GEM_BUG_ON(!(flags & __EXEC_OBJECT_HAS_PIN));
 417
 418        if (unlikely(flags & __EXEC_OBJECT_HAS_FENCE))
 419                __i915_vma_unpin_fence(vma);
 420
 421        __i915_vma_unpin(vma);
 422}
 423
 424static inline void
 425eb_unreserve_vma(struct i915_vma *vma, unsigned int *flags)
 426{
 427        if (!(*flags & __EXEC_OBJECT_HAS_PIN))
 428                return;
 429
 430        __eb_unreserve_vma(vma, *flags);
 431        *flags &= ~__EXEC_OBJECT_RESERVED;
 432}
 433
 434static int
 435eb_validate_vma(struct i915_execbuffer *eb,
 436                struct drm_i915_gem_exec_object2 *entry,
 437                struct i915_vma *vma)
 438{
 439        if (unlikely(entry->flags & eb->invalid_flags))
 440                return -EINVAL;
 441
 442        if (unlikely(entry->alignment && !is_power_of_2(entry->alignment)))
 443                return -EINVAL;
 444
 445        /*
 446         * Offset can be used as input (EXEC_OBJECT_PINNED), reject
 447         * any non-page-aligned or non-canonical addresses.
 448         */
 449        if (unlikely(entry->flags & EXEC_OBJECT_PINNED &&
 450                     entry->offset != gen8_canonical_addr(entry->offset & I915_GTT_PAGE_MASK)))
 451                return -EINVAL;
 452
 453        /* pad_to_size was once a reserved field, so sanitize it */
 454        if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) {
 455                if (unlikely(offset_in_page(entry->pad_to_size)))
 456                        return -EINVAL;
 457        } else {
 458                entry->pad_to_size = 0;
 459        }
 460
 461        if (unlikely(vma->exec_flags)) {
 462                DRM_DEBUG("Object [handle %d, index %d] appears more than once in object list\n",
 463                          entry->handle, (int)(entry - eb->exec));
 464                return -EINVAL;
 465        }
 466
 467        /*
 468         * From drm_mm perspective address space is continuous,
 469         * so from this point we're always using non-canonical
 470         * form internally.
 471         */
 472        entry->offset = gen8_noncanonical_addr(entry->offset);
 473
 474        if (!eb->reloc_cache.has_fence) {
 475                entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
 476        } else {
 477                if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE ||
 478                     eb->reloc_cache.needs_unfenced) &&
 479                    i915_gem_object_is_tiled(vma->obj))
 480                        entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP;
 481        }
 482
 483        if (!(entry->flags & EXEC_OBJECT_PINNED))
 484                entry->flags |= eb->context_flags;
 485
 486        return 0;
 487}
 488
 489static int
 490eb_add_vma(struct i915_execbuffer *eb,
 491           unsigned int i, unsigned batch_idx,
 492           struct i915_vma *vma)
 493{
 494        struct drm_i915_gem_exec_object2 *entry = &eb->exec[i];
 495        int err;
 496
 497        GEM_BUG_ON(i915_vma_is_closed(vma));
 498
 499        if (!(eb->args->flags & __EXEC_VALIDATED)) {
 500                err = eb_validate_vma(eb, entry, vma);
 501                if (unlikely(err))
 502                        return err;
 503        }
 504
 505        if (eb->lut_size > 0) {
 506                vma->exec_handle = entry->handle;
 507                hlist_add_head(&vma->exec_node,
 508                               &eb->buckets[hash_32(entry->handle,
 509                                                    eb->lut_size)]);
 510        }
 511
 512        if (entry->relocation_count)
 513                list_add_tail(&vma->reloc_link, &eb->relocs);
 514
 515        /*
 516         * Stash a pointer from the vma to execobj, so we can query its flags,
 517         * size, alignment etc as provided by the user. Also we stash a pointer
 518         * to the vma inside the execobj so that we can use a direct lookup
 519         * to find the right target VMA when doing relocations.
 520         */
 521        eb->vma[i] = vma;
 522        eb->flags[i] = entry->flags;
 523        vma->exec_flags = &eb->flags[i];
 524
 525        /*
 526         * SNA is doing fancy tricks with compressing batch buffers, which leads
 527         * to negative relocation deltas. Usually that works out ok since the
 528         * relocate address is still positive, except when the batch is placed
 529         * very low in the GTT. Ensure this doesn't happen.
 530         *
 531         * Note that actual hangs have only been observed on gen7, but for
 532         * paranoia do it everywhere.
 533         */
 534        if (i == batch_idx) {
 535                if (entry->relocation_count &&
 536                    !(eb->flags[i] & EXEC_OBJECT_PINNED))
 537                        eb->flags[i] |= __EXEC_OBJECT_NEEDS_BIAS;
 538                if (eb->reloc_cache.has_fence)
 539                        eb->flags[i] |= EXEC_OBJECT_NEEDS_FENCE;
 540
 541                eb->batch = vma;
 542        }
 543
 544        err = 0;
 545        if (eb_pin_vma(eb, entry, vma)) {
 546                if (entry->offset != vma->node.start) {
 547                        entry->offset = vma->node.start | UPDATE;
 548                        eb->args->flags |= __EXEC_HAS_RELOC;
 549                }
 550        } else {
 551                eb_unreserve_vma(vma, vma->exec_flags);
 552
 553                list_add_tail(&vma->exec_link, &eb->unbound);
 554                if (drm_mm_node_allocated(&vma->node))
 555                        err = i915_vma_unbind(vma);
 556                if (unlikely(err))
 557                        vma->exec_flags = NULL;
 558        }
 559        return err;
 560}
 561
 562static inline int use_cpu_reloc(const struct reloc_cache *cache,
 563                                const struct drm_i915_gem_object *obj)
 564{
 565        if (!i915_gem_object_has_struct_page(obj))
 566                return false;
 567
 568        if (DBG_FORCE_RELOC == FORCE_CPU_RELOC)
 569                return true;
 570
 571        if (DBG_FORCE_RELOC == FORCE_GTT_RELOC)
 572                return false;
 573
 574        return (cache->has_llc ||
 575                obj->cache_dirty ||
 576                obj->cache_level != I915_CACHE_NONE);
 577}
 578
 579static int eb_reserve_vma(const struct i915_execbuffer *eb,
 580                          struct i915_vma *vma)
 581{
 582        struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma);
 583        unsigned int exec_flags = *vma->exec_flags;
 584        u64 pin_flags;
 585        int err;
 586
 587        pin_flags = PIN_USER | PIN_NONBLOCK;
 588        if (exec_flags & EXEC_OBJECT_NEEDS_GTT)
 589                pin_flags |= PIN_GLOBAL;
 590
 591        /*
 592         * Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset,
 593         * limit address to the first 4GBs for unflagged objects.
 594         */
 595        if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
 596                pin_flags |= PIN_ZONE_4G;
 597
 598        if (exec_flags & __EXEC_OBJECT_NEEDS_MAP)
 599                pin_flags |= PIN_MAPPABLE;
 600
 601        if (exec_flags & EXEC_OBJECT_PINNED) {
 602                pin_flags |= entry->offset | PIN_OFFSET_FIXED;
 603                pin_flags &= ~PIN_NONBLOCK; /* force overlapping checks */
 604        } else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS) {
 605                pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS;
 606        }
 607
 608        err = i915_vma_pin(vma,
 609                           entry->pad_to_size, entry->alignment,
 610                           pin_flags);
 611        if (err)
 612                return err;
 613
 614        if (entry->offset != vma->node.start) {
 615                entry->offset = vma->node.start | UPDATE;
 616                eb->args->flags |= __EXEC_HAS_RELOC;
 617        }
 618
 619        if (unlikely(exec_flags & EXEC_OBJECT_NEEDS_FENCE)) {
 620                err = i915_vma_pin_fence(vma);
 621                if (unlikely(err)) {
 622                        i915_vma_unpin(vma);
 623                        return err;
 624                }
 625
 626                if (vma->fence)
 627                        exec_flags |= __EXEC_OBJECT_HAS_FENCE;
 628        }
 629
 630        *vma->exec_flags = exec_flags | __EXEC_OBJECT_HAS_PIN;
 631        GEM_BUG_ON(eb_vma_misplaced(entry, vma, exec_flags));
 632
 633        return 0;
 634}
 635
 636static int eb_reserve(struct i915_execbuffer *eb)
 637{
 638        const unsigned int count = eb->buffer_count;
 639        struct list_head last;
 640        struct i915_vma *vma;
 641        unsigned int i, pass;
 642        int err;
 643
 644        /*
 645         * Attempt to pin all of the buffers into the GTT.
 646         * This is done in 3 phases:
 647         *
 648         * 1a. Unbind all objects that do not match the GTT constraints for
 649         *     the execbuffer (fenceable, mappable, alignment etc).
 650         * 1b. Increment pin count for already bound objects.
 651         * 2.  Bind new objects.
 652         * 3.  Decrement pin count.
 653         *
 654         * This avoid unnecessary unbinding of later objects in order to make
 655         * room for the earlier objects *unless* we need to defragment.
 656         */
 657
 658        pass = 0;
 659        err = 0;
 660        do {
 661                list_for_each_entry(vma, &eb->unbound, exec_link) {
 662                        err = eb_reserve_vma(eb, vma);
 663                        if (err)
 664                                break;
 665                }
 666                if (err != -ENOSPC)
 667                        return err;
 668
 669                /* Resort *all* the objects into priority order */
 670                INIT_LIST_HEAD(&eb->unbound);
 671                INIT_LIST_HEAD(&last);
 672                for (i = 0; i < count; i++) {
 673                        unsigned int flags = eb->flags[i];
 674                        struct i915_vma *vma = eb->vma[i];
 675
 676                        if (flags & EXEC_OBJECT_PINNED &&
 677                            flags & __EXEC_OBJECT_HAS_PIN)
 678                                continue;
 679
 680                        eb_unreserve_vma(vma, &eb->flags[i]);
 681
 682                        if (flags & EXEC_OBJECT_PINNED)
 683                                /* Pinned must have their slot */
 684                                list_add(&vma->exec_link, &eb->unbound);
 685                        else if (flags & __EXEC_OBJECT_NEEDS_MAP)
 686                                /* Map require the lowest 256MiB (aperture) */
 687                                list_add_tail(&vma->exec_link, &eb->unbound);
 688                        else if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS))
 689                                /* Prioritise 4GiB region for restricted bo */
 690                                list_add(&vma->exec_link, &last);
 691                        else
 692                                list_add_tail(&vma->exec_link, &last);
 693                }
 694                list_splice_tail(&last, &eb->unbound);
 695
 696                switch (pass++) {
 697                case 0:
 698                        break;
 699
 700                case 1:
 701                        /* Too fragmented, unbind everything and retry */
 702                        err = i915_gem_evict_vm(eb->context->vm);
 703                        if (err)
 704                                return err;
 705                        break;
 706
 707                default:
 708                        return -ENOSPC;
 709                }
 710        } while (1);
 711}
 712
 713static unsigned int eb_batch_index(const struct i915_execbuffer *eb)
 714{
 715        if (eb->args->flags & I915_EXEC_BATCH_FIRST)
 716                return 0;
 717        else
 718                return eb->buffer_count - 1;
 719}
 720
 721static int eb_select_context(struct i915_execbuffer *eb)
 722{
 723        struct i915_gem_context *ctx;
 724
 725        ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1);
 726        if (unlikely(!ctx))
 727                return -ENOENT;
 728
 729        eb->gem_context = ctx;
 730        if (ctx->vm)
 731                eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
 732
 733        eb->context_flags = 0;
 734        if (test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags))
 735                eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS;
 736
 737        return 0;
 738}
 739
 740static int eb_lookup_vmas(struct i915_execbuffer *eb)
 741{
 742        struct radix_tree_root *handles_vma = &eb->gem_context->handles_vma;
 743        struct drm_i915_gem_object *obj;
 744        unsigned int i, batch;
 745        int err;
 746
 747        if (unlikely(i915_gem_context_is_banned(eb->gem_context)))
 748                return -EIO;
 749
 750        INIT_LIST_HEAD(&eb->relocs);
 751        INIT_LIST_HEAD(&eb->unbound);
 752
 753        batch = eb_batch_index(eb);
 754
 755        mutex_lock(&eb->gem_context->mutex);
 756        if (unlikely(i915_gem_context_is_closed(eb->gem_context))) {
 757                err = -ENOENT;
 758                goto err_ctx;
 759        }
 760
 761        for (i = 0; i < eb->buffer_count; i++) {
 762                u32 handle = eb->exec[i].handle;
 763                struct i915_lut_handle *lut;
 764                struct i915_vma *vma;
 765
 766                vma = radix_tree_lookup(handles_vma, handle);
 767                if (likely(vma))
 768                        goto add_vma;
 769
 770                obj = i915_gem_object_lookup(eb->file, handle);
 771                if (unlikely(!obj)) {
 772                        err = -ENOENT;
 773                        goto err_vma;
 774                }
 775
 776                vma = i915_vma_instance(obj, eb->context->vm, NULL);
 777                if (IS_ERR(vma)) {
 778                        err = PTR_ERR(vma);
 779                        goto err_obj;
 780                }
 781
 782                lut = i915_lut_handle_alloc();
 783                if (unlikely(!lut)) {
 784                        err = -ENOMEM;
 785                        goto err_obj;
 786                }
 787
 788                err = radix_tree_insert(handles_vma, handle, vma);
 789                if (unlikely(err)) {
 790                        i915_lut_handle_free(lut);
 791                        goto err_obj;
 792                }
 793
 794                /* transfer ref to lut */
 795                if (!atomic_fetch_inc(&vma->open_count))
 796                        i915_vma_reopen(vma);
 797                lut->handle = handle;
 798                lut->ctx = eb->gem_context;
 799
 800                i915_gem_object_lock(obj);
 801                list_add(&lut->obj_link, &obj->lut_list);
 802                i915_gem_object_unlock(obj);
 803
 804add_vma:
 805                err = eb_add_vma(eb, i, batch, vma);
 806                if (unlikely(err))
 807                        goto err_vma;
 808
 809                GEM_BUG_ON(vma != eb->vma[i]);
 810                GEM_BUG_ON(vma->exec_flags != &eb->flags[i]);
 811                GEM_BUG_ON(drm_mm_node_allocated(&vma->node) &&
 812                           eb_vma_misplaced(&eb->exec[i], vma, eb->flags[i]));
 813        }
 814
 815        mutex_unlock(&eb->gem_context->mutex);
 816
 817        eb->args->flags |= __EXEC_VALIDATED;
 818        return eb_reserve(eb);
 819
 820err_obj:
 821        i915_gem_object_put(obj);
 822err_vma:
 823        eb->vma[i] = NULL;
 824err_ctx:
 825        mutex_unlock(&eb->gem_context->mutex);
 826        return err;
 827}
 828
 829static struct i915_vma *
 830eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle)
 831{
 832        if (eb->lut_size < 0) {
 833                if (handle >= -eb->lut_size)
 834                        return NULL;
 835                return eb->vma[handle];
 836        } else {
 837                struct hlist_head *head;
 838                struct i915_vma *vma;
 839
 840                head = &eb->buckets[hash_32(handle, eb->lut_size)];
 841                hlist_for_each_entry(vma, head, exec_node) {
 842                        if (vma->exec_handle == handle)
 843                                return vma;
 844                }
 845                return NULL;
 846        }
 847}
 848
 849static void eb_release_vmas(const struct i915_execbuffer *eb)
 850{
 851        const unsigned int count = eb->buffer_count;
 852        unsigned int i;
 853
 854        for (i = 0; i < count; i++) {
 855                struct i915_vma *vma = eb->vma[i];
 856                unsigned int flags = eb->flags[i];
 857
 858                if (!vma)
 859                        break;
 860
 861                GEM_BUG_ON(vma->exec_flags != &eb->flags[i]);
 862                vma->exec_flags = NULL;
 863                eb->vma[i] = NULL;
 864
 865                if (flags & __EXEC_OBJECT_HAS_PIN)
 866                        __eb_unreserve_vma(vma, flags);
 867
 868                if (flags & __EXEC_OBJECT_HAS_REF)
 869                        i915_vma_put(vma);
 870        }
 871}
 872
 873static void eb_reset_vmas(const struct i915_execbuffer *eb)
 874{
 875        eb_release_vmas(eb);
 876        if (eb->lut_size > 0)
 877                memset(eb->buckets, 0,
 878                       sizeof(struct hlist_head) << eb->lut_size);
 879}
 880
 881static void eb_destroy(const struct i915_execbuffer *eb)
 882{
 883        GEM_BUG_ON(eb->reloc_cache.rq);
 884
 885        if (eb->lut_size > 0)
 886                kfree(eb->buckets);
 887}
 888
 889static inline u64
 890relocation_target(const struct drm_i915_gem_relocation_entry *reloc,
 891                  const struct i915_vma *target)
 892{
 893        return gen8_canonical_addr((int)reloc->delta + target->node.start);
 894}
 895
 896static void reloc_cache_init(struct reloc_cache *cache,
 897                             struct drm_i915_private *i915)
 898{
 899        cache->page = -1;
 900        cache->vaddr = 0;
 901        /* Must be a variable in the struct to allow GCC to unroll. */
 902        cache->gen = INTEL_GEN(i915);
 903        cache->has_llc = HAS_LLC(i915);
 904        cache->use_64bit_reloc = HAS_64BIT_RELOC(i915);
 905        cache->has_fence = cache->gen < 4;
 906        cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment;
 907        cache->node.allocated = false;
 908        cache->rq = NULL;
 909        cache->rq_size = 0;
 910}
 911
 912static inline void *unmask_page(unsigned long p)
 913{
 914        return (void *)(uintptr_t)(p & PAGE_MASK);
 915}
 916
 917static inline unsigned int unmask_flags(unsigned long p)
 918{
 919        return p & ~PAGE_MASK;
 920}
 921
 922#define KMAP 0x4 /* after CLFLUSH_FLAGS */
 923
 924static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache)
 925{
 926        struct drm_i915_private *i915 =
 927                container_of(cache, struct i915_execbuffer, reloc_cache)->i915;
 928        return &i915->ggtt;
 929}
 930
 931static void reloc_gpu_flush(struct reloc_cache *cache)
 932{
 933        GEM_BUG_ON(cache->rq_size >= cache->rq->batch->obj->base.size / sizeof(u32));
 934        cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END;
 935
 936        __i915_gem_object_flush_map(cache->rq->batch->obj, 0, cache->rq_size);
 937        i915_gem_object_unpin_map(cache->rq->batch->obj);
 938
 939        intel_gt_chipset_flush(cache->rq->engine->gt);
 940
 941        i915_request_add(cache->rq);
 942        cache->rq = NULL;
 943}
 944
 945static void reloc_cache_reset(struct reloc_cache *cache)
 946{
 947        void *vaddr;
 948
 949        if (cache->rq)
 950                reloc_gpu_flush(cache);
 951
 952        if (!cache->vaddr)
 953                return;
 954
 955        vaddr = unmask_page(cache->vaddr);
 956        if (cache->vaddr & KMAP) {
 957                if (cache->vaddr & CLFLUSH_AFTER)
 958                        mb();
 959
 960                kunmap_atomic(vaddr);
 961                i915_gem_object_finish_access((struct drm_i915_gem_object *)cache->node.mm);
 962        } else {
 963                struct i915_ggtt *ggtt = cache_to_ggtt(cache);
 964
 965                intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 966                io_mapping_unmap_atomic((void __iomem *)vaddr);
 967
 968                if (cache->node.allocated) {
 969                        ggtt->vm.clear_range(&ggtt->vm,
 970                                             cache->node.start,
 971                                             cache->node.size);
 972                        drm_mm_remove_node(&cache->node);
 973                } else {
 974                        i915_vma_unpin((struct i915_vma *)cache->node.mm);
 975                }
 976        }
 977
 978        cache->vaddr = 0;
 979        cache->page = -1;
 980}
 981
 982static void *reloc_kmap(struct drm_i915_gem_object *obj,
 983                        struct reloc_cache *cache,
 984                        unsigned long page)
 985{
 986        void *vaddr;
 987
 988        if (cache->vaddr) {
 989                kunmap_atomic(unmask_page(cache->vaddr));
 990        } else {
 991                unsigned int flushes;
 992                int err;
 993
 994                err = i915_gem_object_prepare_write(obj, &flushes);
 995                if (err)
 996                        return ERR_PTR(err);
 997
 998                BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS);
 999                BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK);
1000
1001                cache->vaddr = flushes | KMAP;
1002                cache->node.mm = (void *)obj;
1003                if (flushes)
1004                        mb();
1005        }
1006
1007        vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj, page));
1008        cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr;
1009        cache->page = page;
1010
1011        return vaddr;
1012}
1013
1014static void *reloc_iomap(struct drm_i915_gem_object *obj,
1015                         struct reloc_cache *cache,
1016                         unsigned long page)
1017{
1018        struct i915_ggtt *ggtt = cache_to_ggtt(cache);
1019        unsigned long offset;
1020        void *vaddr;
1021
1022        if (cache->vaddr) {
1023                intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1024                io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr));
1025        } else {
1026                struct i915_vma *vma;
1027                int err;
1028
1029                if (i915_gem_object_is_tiled(obj))
1030                        return ERR_PTR(-EINVAL);
1031
1032                if (use_cpu_reloc(cache, obj))
1033                        return NULL;
1034
1035                i915_gem_object_lock(obj);
1036                err = i915_gem_object_set_to_gtt_domain(obj, true);
1037                i915_gem_object_unlock(obj);
1038                if (err)
1039                        return ERR_PTR(err);
1040
1041                vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1042                                               PIN_MAPPABLE |
1043                                               PIN_NONBLOCK /* NOWARN */ |
1044                                               PIN_NOEVICT);
1045                if (IS_ERR(vma)) {
1046                        memset(&cache->node, 0, sizeof(cache->node));
1047                        err = drm_mm_insert_node_in_range
1048                                (&ggtt->vm.mm, &cache->node,
1049                                 PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE,
1050                                 0, ggtt->mappable_end,
1051                                 DRM_MM_INSERT_LOW);
1052                        if (err) /* no inactive aperture space, use cpu reloc */
1053                                return NULL;
1054                } else {
1055                        cache->node.start = vma->node.start;
1056                        cache->node.mm = (void *)vma;
1057                }
1058        }
1059
1060        offset = cache->node.start;
1061        if (cache->node.allocated) {
1062                ggtt->vm.insert_page(&ggtt->vm,
1063                                     i915_gem_object_get_dma_address(obj, page),
1064                                     offset, I915_CACHE_NONE, 0);
1065        } else {
1066                offset += page << PAGE_SHIFT;
1067        }
1068
1069        vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap,
1070                                                         offset);
1071        cache->page = page;
1072        cache->vaddr = (unsigned long)vaddr;
1073
1074        return vaddr;
1075}
1076
1077static void *reloc_vaddr(struct drm_i915_gem_object *obj,
1078                         struct reloc_cache *cache,
1079                         unsigned long page)
1080{
1081        void *vaddr;
1082
1083        if (cache->page == page) {
1084                vaddr = unmask_page(cache->vaddr);
1085        } else {
1086                vaddr = NULL;
1087                if ((cache->vaddr & KMAP) == 0)
1088                        vaddr = reloc_iomap(obj, cache, page);
1089                if (!vaddr)
1090                        vaddr = reloc_kmap(obj, cache, page);
1091        }
1092
1093        return vaddr;
1094}
1095
1096static void clflush_write32(u32 *addr, u32 value, unsigned int flushes)
1097{
1098        if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) {
1099                if (flushes & CLFLUSH_BEFORE) {
1100                        clflushopt(addr);
1101                        mb();
1102                }
1103
1104                *addr = value;
1105
1106                /*
1107                 * Writes to the same cacheline are serialised by the CPU
1108                 * (including clflush). On the write path, we only require
1109                 * that it hits memory in an orderly fashion and place
1110                 * mb barriers at the start and end of the relocation phase
1111                 * to ensure ordering of clflush wrt to the system.
1112                 */
1113                if (flushes & CLFLUSH_AFTER)
1114                        clflushopt(addr);
1115        } else
1116                *addr = value;
1117}
1118
1119static int reloc_move_to_gpu(struct i915_request *rq, struct i915_vma *vma)
1120{
1121        struct drm_i915_gem_object *obj = vma->obj;
1122        int err;
1123
1124        i915_vma_lock(vma);
1125
1126        if (obj->cache_dirty & ~obj->cache_coherent)
1127                i915_gem_clflush_object(obj, 0);
1128        obj->write_domain = 0;
1129
1130        err = i915_request_await_object(rq, vma->obj, true);
1131        if (err == 0)
1132                err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
1133
1134        i915_vma_unlock(vma);
1135
1136        return err;
1137}
1138
1139static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
1140                             struct i915_vma *vma,
1141                             unsigned int len)
1142{
1143        struct reloc_cache *cache = &eb->reloc_cache;
1144        struct intel_engine_pool_node *pool;
1145        struct i915_request *rq;
1146        struct i915_vma *batch;
1147        u32 *cmd;
1148        int err;
1149
1150        pool = intel_engine_pool_get(&eb->engine->pool, PAGE_SIZE);
1151        if (IS_ERR(pool))
1152                return PTR_ERR(pool);
1153
1154        cmd = i915_gem_object_pin_map(pool->obj,
1155                                      cache->has_llc ?
1156                                      I915_MAP_FORCE_WB :
1157                                      I915_MAP_FORCE_WC);
1158        if (IS_ERR(cmd)) {
1159                err = PTR_ERR(cmd);
1160                goto out_pool;
1161        }
1162
1163        batch = i915_vma_instance(pool->obj, vma->vm, NULL);
1164        if (IS_ERR(batch)) {
1165                err = PTR_ERR(batch);
1166                goto err_unmap;
1167        }
1168
1169        err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK);
1170        if (err)
1171                goto err_unmap;
1172
1173        rq = i915_request_create(eb->context);
1174        if (IS_ERR(rq)) {
1175                err = PTR_ERR(rq);
1176                goto err_unpin;
1177        }
1178
1179        err = intel_engine_pool_mark_active(pool, rq);
1180        if (err)
1181                goto err_request;
1182
1183        err = reloc_move_to_gpu(rq, vma);
1184        if (err)
1185                goto err_request;
1186
1187        err = eb->engine->emit_bb_start(rq,
1188                                        batch->node.start, PAGE_SIZE,
1189                                        cache->gen > 5 ? 0 : I915_DISPATCH_SECURE);
1190        if (err)
1191                goto skip_request;
1192
1193        i915_vma_lock(batch);
1194        err = i915_request_await_object(rq, batch->obj, false);
1195        if (err == 0)
1196                err = i915_vma_move_to_active(batch, rq, 0);
1197        i915_vma_unlock(batch);
1198        if (err)
1199                goto skip_request;
1200
1201        rq->batch = batch;
1202        i915_vma_unpin(batch);
1203
1204        cache->rq = rq;
1205        cache->rq_cmd = cmd;
1206        cache->rq_size = 0;
1207
1208        /* Return with batch mapping (cmd) still pinned */
1209        goto out_pool;
1210
1211skip_request:
1212        i915_request_skip(rq, err);
1213err_request:
1214        i915_request_add(rq);
1215err_unpin:
1216        i915_vma_unpin(batch);
1217err_unmap:
1218        i915_gem_object_unpin_map(pool->obj);
1219out_pool:
1220        intel_engine_pool_put(pool);
1221        return err;
1222}
1223
1224static u32 *reloc_gpu(struct i915_execbuffer *eb,
1225                      struct i915_vma *vma,
1226                      unsigned int len)
1227{
1228        struct reloc_cache *cache = &eb->reloc_cache;
1229        u32 *cmd;
1230
1231        if (cache->rq_size > PAGE_SIZE/sizeof(u32) - (len + 1))
1232                reloc_gpu_flush(cache);
1233
1234        if (unlikely(!cache->rq)) {
1235                int err;
1236
1237                /* If we need to copy for the cmdparser, we will stall anyway */
1238                if (eb_use_cmdparser(eb))
1239                        return ERR_PTR(-EWOULDBLOCK);
1240
1241                if (!intel_engine_can_store_dword(eb->engine))
1242                        return ERR_PTR(-ENODEV);
1243
1244                err = __reloc_gpu_alloc(eb, vma, len);
1245                if (unlikely(err))
1246                        return ERR_PTR(err);
1247        }
1248
1249        cmd = cache->rq_cmd + cache->rq_size;
1250        cache->rq_size += len;
1251
1252        return cmd;
1253}
1254
1255static u64
1256relocate_entry(struct i915_vma *vma,
1257               const struct drm_i915_gem_relocation_entry *reloc,
1258               struct i915_execbuffer *eb,
1259               const struct i915_vma *target)
1260{
1261        u64 offset = reloc->offset;
1262        u64 target_offset = relocation_target(reloc, target);
1263        bool wide = eb->reloc_cache.use_64bit_reloc;
1264        void *vaddr;
1265
1266        if (!eb->reloc_cache.vaddr &&
1267            (DBG_FORCE_RELOC == FORCE_GPU_RELOC ||
1268             !dma_resv_test_signaled_rcu(vma->resv, true))) {
1269                const unsigned int gen = eb->reloc_cache.gen;
1270                unsigned int len;
1271                u32 *batch;
1272                u64 addr;
1273
1274                if (wide)
1275                        len = offset & 7 ? 8 : 5;
1276                else if (gen >= 4)
1277                        len = 4;
1278                else
1279                        len = 3;
1280
1281                batch = reloc_gpu(eb, vma, len);
1282                if (IS_ERR(batch))
1283                        goto repeat;
1284
1285                addr = gen8_canonical_addr(vma->node.start + offset);
1286                if (wide) {
1287                        if (offset & 7) {
1288                                *batch++ = MI_STORE_DWORD_IMM_GEN4;
1289                                *batch++ = lower_32_bits(addr);
1290                                *batch++ = upper_32_bits(addr);
1291                                *batch++ = lower_32_bits(target_offset);
1292
1293                                addr = gen8_canonical_addr(addr + 4);
1294
1295                                *batch++ = MI_STORE_DWORD_IMM_GEN4;
1296                                *batch++ = lower_32_bits(addr);
1297                                *batch++ = upper_32_bits(addr);
1298                                *batch++ = upper_32_bits(target_offset);
1299                        } else {
1300                                *batch++ = (MI_STORE_DWORD_IMM_GEN4 | (1 << 21)) + 1;
1301                                *batch++ = lower_32_bits(addr);
1302                                *batch++ = upper_32_bits(addr);
1303                                *batch++ = lower_32_bits(target_offset);
1304                                *batch++ = upper_32_bits(target_offset);
1305                        }
1306                } else if (gen >= 6) {
1307                        *batch++ = MI_STORE_DWORD_IMM_GEN4;
1308                        *batch++ = 0;
1309                        *batch++ = addr;
1310                        *batch++ = target_offset;
1311                } else if (gen >= 4) {
1312                        *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1313                        *batch++ = 0;
1314                        *batch++ = addr;
1315                        *batch++ = target_offset;
1316                } else {
1317                        *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
1318                        *batch++ = addr;
1319                        *batch++ = target_offset;
1320                }
1321
1322                goto out;
1323        }
1324
1325repeat:
1326        vaddr = reloc_vaddr(vma->obj, &eb->reloc_cache, offset >> PAGE_SHIFT);
1327        if (IS_ERR(vaddr))
1328                return PTR_ERR(vaddr);
1329
1330        clflush_write32(vaddr + offset_in_page(offset),
1331                        lower_32_bits(target_offset),
1332                        eb->reloc_cache.vaddr);
1333
1334        if (wide) {
1335                offset += sizeof(u32);
1336                target_offset >>= 32;
1337                wide = false;
1338                goto repeat;
1339        }
1340
1341out:
1342        return target->node.start | UPDATE;
1343}
1344
1345static u64
1346eb_relocate_entry(struct i915_execbuffer *eb,
1347                  struct i915_vma *vma,
1348                  const struct drm_i915_gem_relocation_entry *reloc)
1349{
1350        struct i915_vma *target;
1351        int err;
1352
1353        /* we've already hold a reference to all valid objects */
1354        target = eb_get_vma(eb, reloc->target_handle);
1355        if (unlikely(!target))
1356                return -ENOENT;
1357
1358        /* Validate that the target is in a valid r/w GPU domain */
1359        if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) {
1360                DRM_DEBUG("reloc with multiple write domains: "
1361                          "target %d offset %d "
1362                          "read %08x write %08x",
1363                          reloc->target_handle,
1364                          (int) reloc->offset,
1365                          reloc->read_domains,
1366                          reloc->write_domain);
1367                return -EINVAL;
1368        }
1369        if (unlikely((reloc->write_domain | reloc->read_domains)
1370                     & ~I915_GEM_GPU_DOMAINS)) {
1371                DRM_DEBUG("reloc with read/write non-GPU domains: "
1372                          "target %d offset %d "
1373                          "read %08x write %08x",
1374                          reloc->target_handle,
1375                          (int) reloc->offset,
1376                          reloc->read_domains,
1377                          reloc->write_domain);
1378                return -EINVAL;
1379        }
1380
1381        if (reloc->write_domain) {
1382                *target->exec_flags |= EXEC_OBJECT_WRITE;
1383
1384                /*
1385                 * Sandybridge PPGTT errata: We need a global gtt mapping
1386                 * for MI and pipe_control writes because the gpu doesn't
1387                 * properly redirect them through the ppgtt for non_secure
1388                 * batchbuffers.
1389                 */
1390                if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
1391                    IS_GEN(eb->i915, 6)) {
1392                        err = i915_vma_bind(target, target->obj->cache_level,
1393                                            PIN_GLOBAL);
1394                        if (WARN_ONCE(err,
1395                                      "Unexpected failure to bind target VMA!"))
1396                                return err;
1397                }
1398        }
1399
1400        /*
1401         * If the relocation already has the right value in it, no
1402         * more work needs to be done.
1403         */
1404        if (!DBG_FORCE_RELOC &&
1405            gen8_canonical_addr(target->node.start) == reloc->presumed_offset)
1406                return 0;
1407
1408        /* Check that the relocation address is valid... */
1409        if (unlikely(reloc->offset >
1410                     vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) {
1411                DRM_DEBUG("Relocation beyond object bounds: "
1412                          "target %d offset %d size %d.\n",
1413                          reloc->target_handle,
1414                          (int)reloc->offset,
1415                          (int)vma->size);
1416                return -EINVAL;
1417        }
1418        if (unlikely(reloc->offset & 3)) {
1419                DRM_DEBUG("Relocation not 4-byte aligned: "
1420                          "target %d offset %d.\n",
1421                          reloc->target_handle,
1422                          (int)reloc->offset);
1423                return -EINVAL;
1424        }
1425
1426        /*
1427         * If we write into the object, we need to force the synchronisation
1428         * barrier, either with an asynchronous clflush or if we executed the
1429         * patching using the GPU (though that should be serialised by the
1430         * timeline). To be completely sure, and since we are required to
1431         * do relocations we are already stalling, disable the user's opt
1432         * out of our synchronisation.
1433         */
1434        *vma->exec_flags &= ~EXEC_OBJECT_ASYNC;
1435
1436        /* and update the user's relocation entry */
1437        return relocate_entry(vma, reloc, eb, target);
1438}
1439
1440static int eb_relocate_vma(struct i915_execbuffer *eb, struct i915_vma *vma)
1441{
1442#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
1443        struct drm_i915_gem_relocation_entry stack[N_RELOC(512)];
1444        struct drm_i915_gem_relocation_entry __user *urelocs;
1445        const struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma);
1446        unsigned int remain;
1447
1448        urelocs = u64_to_user_ptr(entry->relocs_ptr);
1449        remain = entry->relocation_count;
1450        if (unlikely(remain > N_RELOC(ULONG_MAX)))
1451                return -EINVAL;
1452
1453        /*
1454         * We must check that the entire relocation array is safe
1455         * to read. However, if the array is not writable the user loses
1456         * the updated relocation values.
1457         */
1458        if (unlikely(!access_ok(urelocs, remain*sizeof(*urelocs))))
1459                return -EFAULT;
1460
1461        do {
1462                struct drm_i915_gem_relocation_entry *r = stack;
1463                unsigned int count =
1464                        min_t(unsigned int, remain, ARRAY_SIZE(stack));
1465                unsigned int copied;
1466
1467                /*
1468                 * This is the fast path and we cannot handle a pagefault
1469                 * whilst holding the struct mutex lest the user pass in the
1470                 * relocations contained within a mmaped bo. For in such a case
1471                 * we, the page fault handler would call i915_gem_fault() and
1472                 * we would try to acquire the struct mutex again. Obviously
1473                 * this is bad and so lockdep complains vehemently.
1474                 */
1475                pagefault_disable();
1476                copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0]));
1477                pagefault_enable();
1478                if (unlikely(copied)) {
1479                        remain = -EFAULT;
1480                        goto out;
1481                }
1482
1483                remain -= count;
1484                do {
1485                        u64 offset = eb_relocate_entry(eb, vma, r);
1486
1487                        if (likely(offset == 0)) {
1488                        } else if ((s64)offset < 0) {
1489                                remain = (int)offset;
1490                                goto out;
1491                        } else {
1492                                /*
1493                                 * Note that reporting an error now
1494                                 * leaves everything in an inconsistent
1495                                 * state as we have *already* changed
1496                                 * the relocation value inside the
1497                                 * object. As we have not changed the
1498                                 * reloc.presumed_offset or will not
1499                                 * change the execobject.offset, on the
1500                                 * call we may not rewrite the value
1501                                 * inside the object, leaving it
1502                                 * dangling and causing a GPU hang. Unless
1503                                 * userspace dynamically rebuilds the
1504                                 * relocations on each execbuf rather than
1505                                 * presume a static tree.
1506                                 *
1507                                 * We did previously check if the relocations
1508                                 * were writable (access_ok), an error now
1509                                 * would be a strange race with mprotect,
1510                                 * having already demonstrated that we
1511                                 * can read from this userspace address.
1512                                 */
1513                                offset = gen8_canonical_addr(offset & ~UPDATE);
1514                                if (unlikely(__put_user(offset, &urelocs[r-stack].presumed_offset))) {
1515                                        remain = -EFAULT;
1516                                        goto out;
1517                                }
1518                        }
1519                } while (r++, --count);
1520                urelocs += ARRAY_SIZE(stack);
1521        } while (remain);
1522out:
1523        reloc_cache_reset(&eb->reloc_cache);
1524        return remain;
1525}
1526
1527static int
1528eb_relocate_vma_slow(struct i915_execbuffer *eb, struct i915_vma *vma)
1529{
1530        const struct drm_i915_gem_exec_object2 *entry = exec_entry(eb, vma);
1531        struct drm_i915_gem_relocation_entry *relocs =
1532                u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
1533        unsigned int i;
1534        int err;
1535
1536        for (i = 0; i < entry->relocation_count; i++) {
1537                u64 offset = eb_relocate_entry(eb, vma, &relocs[i]);
1538
1539                if ((s64)offset < 0) {
1540                        err = (int)offset;
1541                        goto err;
1542                }
1543        }
1544        err = 0;
1545err:
1546        reloc_cache_reset(&eb->reloc_cache);
1547        return err;
1548}
1549
1550static int check_relocations(const struct drm_i915_gem_exec_object2 *entry)
1551{
1552        const char __user *addr, *end;
1553        unsigned long size;
1554        char __maybe_unused c;
1555
1556        size = entry->relocation_count;
1557        if (size == 0)
1558                return 0;
1559
1560        if (size > N_RELOC(ULONG_MAX))
1561                return -EINVAL;
1562
1563        addr = u64_to_user_ptr(entry->relocs_ptr);
1564        size *= sizeof(struct drm_i915_gem_relocation_entry);
1565        if (!access_ok(addr, size))
1566                return -EFAULT;
1567
1568        end = addr + size;
1569        for (; addr < end; addr += PAGE_SIZE) {
1570                int err = __get_user(c, addr);
1571                if (err)
1572                        return err;
1573        }
1574        return __get_user(c, end - 1);
1575}
1576
1577static int eb_copy_relocations(const struct i915_execbuffer *eb)
1578{
1579        struct drm_i915_gem_relocation_entry *relocs;
1580        const unsigned int count = eb->buffer_count;
1581        unsigned int i;
1582        int err;
1583
1584        for (i = 0; i < count; i++) {
1585                const unsigned int nreloc = eb->exec[i].relocation_count;
1586                struct drm_i915_gem_relocation_entry __user *urelocs;
1587                unsigned long size;
1588                unsigned long copied;
1589
1590                if (nreloc == 0)
1591                        continue;
1592
1593                err = check_relocations(&eb->exec[i]);
1594                if (err)
1595                        goto err;
1596
1597                urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr);
1598                size = nreloc * sizeof(*relocs);
1599
1600                relocs = kvmalloc_array(size, 1, GFP_KERNEL);
1601                if (!relocs) {
1602                        err = -ENOMEM;
1603                        goto err;
1604                }
1605
1606                /* copy_from_user is limited to < 4GiB */
1607                copied = 0;
1608                do {
1609                        unsigned int len =
1610                                min_t(u64, BIT_ULL(31), size - copied);
1611
1612                        if (__copy_from_user((char *)relocs + copied,
1613                                             (char __user *)urelocs + copied,
1614                                             len))
1615                                goto end;
1616
1617                        copied += len;
1618                } while (copied < size);
1619
1620                /*
1621                 * As we do not update the known relocation offsets after
1622                 * relocating (due to the complexities in lock handling),
1623                 * we need to mark them as invalid now so that we force the
1624                 * relocation processing next time. Just in case the target
1625                 * object is evicted and then rebound into its old
1626                 * presumed_offset before the next execbuffer - if that
1627                 * happened we would make the mistake of assuming that the
1628                 * relocations were valid.
1629                 */
1630                if (!user_access_begin(urelocs, size))
1631                        goto end;
1632
1633                for (copied = 0; copied < nreloc; copied++)
1634                        unsafe_put_user(-1,
1635                                        &urelocs[copied].presumed_offset,
1636                                        end_user);
1637                user_access_end();
1638
1639                eb->exec[i].relocs_ptr = (uintptr_t)relocs;
1640        }
1641
1642        return 0;
1643
1644end_user:
1645        user_access_end();
1646end:
1647        kvfree(relocs);
1648        err = -EFAULT;
1649err:
1650        while (i--) {
1651                relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr);
1652                if (eb->exec[i].relocation_count)
1653                        kvfree(relocs);
1654        }
1655        return err;
1656}
1657
1658static int eb_prefault_relocations(const struct i915_execbuffer *eb)
1659{
1660        const unsigned int count = eb->buffer_count;
1661        unsigned int i;
1662
1663        if (unlikely(i915_modparams.prefault_disable))
1664                return 0;
1665
1666        for (i = 0; i < count; i++) {
1667                int err;
1668
1669                err = check_relocations(&eb->exec[i]);
1670                if (err)
1671                        return err;
1672        }
1673
1674        return 0;
1675}
1676
1677static noinline int eb_relocate_slow(struct i915_execbuffer *eb)
1678{
1679        struct drm_device *dev = &eb->i915->drm;
1680        bool have_copy = false;
1681        struct i915_vma *vma;
1682        int err = 0;
1683
1684repeat:
1685        if (signal_pending(current)) {
1686                err = -ERESTARTSYS;
1687                goto out;
1688        }
1689
1690        /* We may process another execbuffer during the unlock... */
1691        eb_reset_vmas(eb);
1692        mutex_unlock(&dev->struct_mutex);
1693
1694        /*
1695         * We take 3 passes through the slowpatch.
1696         *
1697         * 1 - we try to just prefault all the user relocation entries and
1698         * then attempt to reuse the atomic pagefault disabled fast path again.
1699         *
1700         * 2 - we copy the user entries to a local buffer here outside of the
1701         * local and allow ourselves to wait upon any rendering before
1702         * relocations
1703         *
1704         * 3 - we already have a local copy of the relocation entries, but
1705         * were interrupted (EAGAIN) whilst waiting for the objects, try again.
1706         */
1707        if (!err) {
1708                err = eb_prefault_relocations(eb);
1709        } else if (!have_copy) {
1710                err = eb_copy_relocations(eb);
1711                have_copy = err == 0;
1712        } else {
1713                cond_resched();
1714                err = 0;
1715        }
1716        if (err) {
1717                mutex_lock(&dev->struct_mutex);
1718                goto out;
1719        }
1720
1721        /* A frequent cause for EAGAIN are currently unavailable client pages */
1722        flush_workqueue(eb->i915->mm.userptr_wq);
1723
1724        err = i915_mutex_lock_interruptible(dev);
1725        if (err) {
1726                mutex_lock(&dev->struct_mutex);
1727                goto out;
1728        }
1729
1730        /* reacquire the objects */
1731        err = eb_lookup_vmas(eb);
1732        if (err)
1733                goto err;
1734
1735        GEM_BUG_ON(!eb->batch);
1736
1737        list_for_each_entry(vma, &eb->relocs, reloc_link) {
1738                if (!have_copy) {
1739                        pagefault_disable();
1740                        err = eb_relocate_vma(eb, vma);
1741                        pagefault_enable();
1742                        if (err)
1743                                goto repeat;
1744                } else {
1745                        err = eb_relocate_vma_slow(eb, vma);
1746                        if (err)
1747                                goto err;
1748                }
1749        }
1750
1751        /*
1752         * Leave the user relocations as are, this is the painfully slow path,
1753         * and we want to avoid the complication of dropping the lock whilst
1754         * having buffers reserved in the aperture and so causing spurious
1755         * ENOSPC for random operations.
1756         */
1757
1758err:
1759        if (err == -EAGAIN)
1760                goto repeat;
1761
1762out:
1763        if (have_copy) {
1764                const unsigned int count = eb->buffer_count;
1765                unsigned int i;
1766
1767                for (i = 0; i < count; i++) {
1768                        const struct drm_i915_gem_exec_object2 *entry =
1769                                &eb->exec[i];
1770                        struct drm_i915_gem_relocation_entry *relocs;
1771
1772                        if (!entry->relocation_count)
1773                                continue;
1774
1775                        relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr);
1776                        kvfree(relocs);
1777                }
1778        }
1779
1780        return err;
1781}
1782
1783static int eb_relocate(struct i915_execbuffer *eb)
1784{
1785        if (eb_lookup_vmas(eb))
1786                goto slow;
1787
1788        /* The objects are in their final locations, apply the relocations. */
1789        if (eb->args->flags & __EXEC_HAS_RELOC) {
1790                struct i915_vma *vma;
1791
1792                list_for_each_entry(vma, &eb->relocs, reloc_link) {
1793                        if (eb_relocate_vma(eb, vma))
1794                                goto slow;
1795                }
1796        }
1797
1798        return 0;
1799
1800slow:
1801        return eb_relocate_slow(eb);
1802}
1803
1804static int eb_move_to_gpu(struct i915_execbuffer *eb)
1805{
1806        const unsigned int count = eb->buffer_count;
1807        struct ww_acquire_ctx acquire;
1808        unsigned int i;
1809        int err = 0;
1810
1811        ww_acquire_init(&acquire, &reservation_ww_class);
1812
1813        for (i = 0; i < count; i++) {
1814                struct i915_vma *vma = eb->vma[i];
1815
1816                err = ww_mutex_lock_interruptible(&vma->resv->lock, &acquire);
1817                if (!err)
1818                        continue;
1819
1820                GEM_BUG_ON(err == -EALREADY); /* No duplicate vma */
1821
1822                if (err == -EDEADLK) {
1823                        GEM_BUG_ON(i == 0);
1824                        do {
1825                                int j = i - 1;
1826
1827                                ww_mutex_unlock(&eb->vma[j]->resv->lock);
1828
1829                                swap(eb->flags[i], eb->flags[j]);
1830                                swap(eb->vma[i],  eb->vma[j]);
1831                                eb->vma[i]->exec_flags = &eb->flags[i];
1832                        } while (--i);
1833                        GEM_BUG_ON(vma != eb->vma[0]);
1834                        vma->exec_flags = &eb->flags[0];
1835
1836                        err = ww_mutex_lock_slow_interruptible(&vma->resv->lock,
1837                                                               &acquire);
1838                }
1839                if (err)
1840                        break;
1841        }
1842        ww_acquire_done(&acquire);
1843
1844        while (i--) {
1845                unsigned int flags = eb->flags[i];
1846                struct i915_vma *vma = eb->vma[i];
1847                struct drm_i915_gem_object *obj = vma->obj;
1848
1849                assert_vma_held(vma);
1850
1851                if (flags & EXEC_OBJECT_CAPTURE) {
1852                        struct i915_capture_list *capture;
1853
1854                        capture = kmalloc(sizeof(*capture), GFP_KERNEL);
1855                        if (capture) {
1856                                capture->next = eb->request->capture_list;
1857                                capture->vma = vma;
1858                                eb->request->capture_list = capture;
1859                        }
1860                }
1861
1862                /*
1863                 * If the GPU is not _reading_ through the CPU cache, we need
1864                 * to make sure that any writes (both previous GPU writes from
1865                 * before a change in snooping levels and normal CPU writes)
1866                 * caught in that cache are flushed to main memory.
1867                 *
1868                 * We want to say
1869                 *   obj->cache_dirty &&
1870                 *   !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ)
1871                 * but gcc's optimiser doesn't handle that as well and emits
1872                 * two jumps instead of one. Maybe one day...
1873                 */
1874                if (unlikely(obj->cache_dirty & ~obj->cache_coherent)) {
1875                        if (i915_gem_clflush_object(obj, 0))
1876                                flags &= ~EXEC_OBJECT_ASYNC;
1877                }
1878
1879                if (err == 0 && !(flags & EXEC_OBJECT_ASYNC)) {
1880                        err = i915_request_await_object
1881                                (eb->request, obj, flags & EXEC_OBJECT_WRITE);
1882                }
1883
1884                if (err == 0)
1885                        err = i915_vma_move_to_active(vma, eb->request, flags);
1886
1887                i915_vma_unlock(vma);
1888
1889                __eb_unreserve_vma(vma, flags);
1890                vma->exec_flags = NULL;
1891
1892                if (unlikely(flags & __EXEC_OBJECT_HAS_REF))
1893                        i915_vma_put(vma);
1894        }
1895        ww_acquire_fini(&acquire);
1896
1897        if (unlikely(err))
1898                goto err_skip;
1899
1900        eb->exec = NULL;
1901
1902        /* Unconditionally flush any chipset caches (for streaming writes). */
1903        intel_gt_chipset_flush(eb->engine->gt);
1904        return 0;
1905
1906err_skip:
1907        i915_request_skip(eb->request, err);
1908        return err;
1909}
1910
1911static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
1912{
1913        if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS)
1914                return false;
1915
1916        /* Kernel clipping was a DRI1 misfeature */
1917        if (!(exec->flags & I915_EXEC_FENCE_ARRAY)) {
1918                if (exec->num_cliprects || exec->cliprects_ptr)
1919                        return false;
1920        }
1921
1922        if (exec->DR4 == 0xffffffff) {
1923                DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
1924                exec->DR4 = 0;
1925        }
1926        if (exec->DR1 || exec->DR4)
1927                return false;
1928
1929        if ((exec->batch_start_offset | exec->batch_len) & 0x7)
1930                return false;
1931
1932        return true;
1933}
1934
1935static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
1936{
1937        u32 *cs;
1938        int i;
1939
1940        if (!IS_GEN(rq->i915, 7) || rq->engine->id != RCS0) {
1941                DRM_DEBUG("sol reset is gen7/rcs only\n");
1942                return -EINVAL;
1943        }
1944
1945        cs = intel_ring_begin(rq, 4 * 2 + 2);
1946        if (IS_ERR(cs))
1947                return PTR_ERR(cs);
1948
1949        *cs++ = MI_LOAD_REGISTER_IMM(4);
1950        for (i = 0; i < 4; i++) {
1951                *cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i));
1952                *cs++ = 0;
1953        }
1954        *cs++ = MI_NOOP;
1955        intel_ring_advance(rq, cs);
1956
1957        return 0;
1958}
1959
1960static struct i915_vma *
1961shadow_batch_pin(struct i915_execbuffer *eb, struct drm_i915_gem_object *obj)
1962{
1963        struct drm_i915_private *dev_priv = eb->i915;
1964        struct i915_vma * const vma = *eb->vma;
1965        struct i915_address_space *vm;
1966        u64 flags;
1967
1968        /*
1969         * PPGTT backed shadow buffers must be mapped RO, to prevent
1970         * post-scan tampering
1971         */
1972        if (CMDPARSER_USES_GGTT(dev_priv)) {
1973                flags = PIN_GLOBAL;
1974                vm = &dev_priv->ggtt.vm;
1975        } else if (vma->vm->has_read_only) {
1976                flags = PIN_USER;
1977                vm = vma->vm;
1978                i915_gem_object_set_readonly(obj);
1979        } else {
1980                DRM_DEBUG("Cannot prevent post-scan tampering without RO capable vm\n");
1981                return ERR_PTR(-EINVAL);
1982        }
1983
1984        return i915_gem_object_pin(obj, vm, NULL, 0, 0, flags);
1985}
1986
1987static struct i915_vma *eb_parse(struct i915_execbuffer *eb)
1988{
1989        struct intel_engine_pool_node *pool;
1990        struct i915_vma *vma;
1991        u64 batch_start;
1992        u64 shadow_batch_start;
1993        int err;
1994
1995        pool = intel_engine_pool_get(&eb->engine->pool, eb->batch_len);
1996        if (IS_ERR(pool))
1997                return ERR_CAST(pool);
1998
1999        vma = shadow_batch_pin(eb, pool->obj);
2000        if (IS_ERR(vma))
2001                goto err;
2002
2003        batch_start = gen8_canonical_addr(eb->batch->node.start) +
2004                      eb->batch_start_offset;
2005
2006        shadow_batch_start = gen8_canonical_addr(vma->node.start);
2007
2008        err = intel_engine_cmd_parser(eb->gem_context,
2009                                      eb->engine,
2010                                      eb->batch->obj,
2011                                      batch_start,
2012                                      eb->batch_start_offset,
2013                                      eb->batch_len,
2014                                      pool->obj,
2015                                      shadow_batch_start);
2016
2017        if (err) {
2018                i915_vma_unpin(vma);
2019
2020                /*
2021                 * Unsafe GGTT-backed buffers can still be submitted safely
2022                 * as non-secure.
2023                 * For PPGTT backing however, we have no choice but to forcibly
2024                 * reject unsafe buffers
2025                 */
2026                if (CMDPARSER_USES_GGTT(eb->i915) && (err == -EACCES))
2027                        /* Execute original buffer non-secure */
2028                        vma = NULL;
2029                else
2030                        vma = ERR_PTR(err);
2031                goto err;
2032        }
2033
2034        eb->vma[eb->buffer_count] = i915_vma_get(vma);
2035        eb->flags[eb->buffer_count] =
2036                __EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_REF;
2037        vma->exec_flags = &eb->flags[eb->buffer_count];
2038        eb->buffer_count++;
2039
2040        eb->batch_start_offset = 0;
2041        eb->batch = vma;
2042
2043        if (CMDPARSER_USES_GGTT(eb->i915))
2044                eb->batch_flags |= I915_DISPATCH_SECURE;
2045
2046        /* eb->batch_len unchanged */
2047
2048        vma->private = pool;
2049        return vma;
2050
2051err:
2052        intel_engine_pool_put(pool);
2053        return vma;
2054}
2055
2056static void
2057add_to_client(struct i915_request *rq, struct drm_file *file)
2058{
2059        struct drm_i915_file_private *file_priv = file->driver_priv;
2060
2061        rq->file_priv = file_priv;
2062
2063        spin_lock(&file_priv->mm.lock);
2064        list_add_tail(&rq->client_link, &file_priv->mm.request_list);
2065        spin_unlock(&file_priv->mm.lock);
2066}
2067
2068static int eb_submit(struct i915_execbuffer *eb)
2069{
2070        int err;
2071
2072        err = eb_move_to_gpu(eb);
2073        if (err)
2074                return err;
2075
2076        if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) {
2077                err = i915_reset_gen7_sol_offsets(eb->request);
2078                if (err)
2079                        return err;
2080        }
2081
2082        /*
2083         * After we completed waiting for other engines (using HW semaphores)
2084         * then we can signal that this request/batch is ready to run. This
2085         * allows us to determine if the batch is still waiting on the GPU
2086         * or actually running by checking the breadcrumb.
2087         */
2088        if (eb->engine->emit_init_breadcrumb) {
2089                err = eb->engine->emit_init_breadcrumb(eb->request);
2090                if (err)
2091                        return err;
2092        }
2093
2094        err = eb->engine->emit_bb_start(eb->request,
2095                                        eb->batch->node.start +
2096                                        eb->batch_start_offset,
2097                                        eb->batch_len,
2098                                        eb->batch_flags);
2099        if (err)
2100                return err;
2101
2102        return 0;
2103}
2104
2105static int num_vcs_engines(const struct drm_i915_private *i915)
2106{
2107        return hweight64(INTEL_INFO(i915)->engine_mask &
2108                         GENMASK_ULL(VCS0 + I915_MAX_VCS - 1, VCS0));
2109}
2110
2111/*
2112 * Find one BSD ring to dispatch the corresponding BSD command.
2113 * The engine index is returned.
2114 */
2115static unsigned int
2116gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv,
2117                         struct drm_file *file)
2118{
2119        struct drm_i915_file_private *file_priv = file->driver_priv;
2120
2121        /* Check whether the file_priv has already selected one ring. */
2122        if ((int)file_priv->bsd_engine < 0)
2123                file_priv->bsd_engine =
2124                        get_random_int() % num_vcs_engines(dev_priv);
2125
2126        return file_priv->bsd_engine;
2127}
2128
2129static const enum intel_engine_id user_ring_map[] = {
2130        [I915_EXEC_DEFAULT]     = RCS0,
2131        [I915_EXEC_RENDER]      = RCS0,
2132        [I915_EXEC_BLT]         = BCS0,
2133        [I915_EXEC_BSD]         = VCS0,
2134        [I915_EXEC_VEBOX]       = VECS0
2135};
2136
2137static struct i915_request *eb_throttle(struct intel_context *ce)
2138{
2139        struct intel_ring *ring = ce->ring;
2140        struct intel_timeline *tl = ce->timeline;
2141        struct i915_request *rq;
2142
2143        /*
2144         * Completely unscientific finger-in-the-air estimates for suitable
2145         * maximum user request size (to avoid blocking) and then backoff.
2146         */
2147        if (intel_ring_update_space(ring) >= PAGE_SIZE)
2148                return NULL;
2149
2150        /*
2151         * Find a request that after waiting upon, there will be at least half
2152         * the ring available. The hysteresis allows us to compete for the
2153         * shared ring and should mean that we sleep less often prior to
2154         * claiming our resources, but not so long that the ring completely
2155         * drains before we can submit our next request.
2156         */
2157        list_for_each_entry(rq, &tl->requests, link) {
2158                if (rq->ring != ring)
2159                        continue;
2160
2161                if (__intel_ring_space(rq->postfix,
2162                                       ring->emit, ring->size) > ring->size / 2)
2163                        break;
2164        }
2165        if (&rq->link == &tl->requests)
2166                return NULL; /* weird, we will check again later for real */
2167
2168        return i915_request_get(rq);
2169}
2170
2171static int
2172__eb_pin_context(struct i915_execbuffer *eb, struct intel_context *ce)
2173{
2174        int err;
2175
2176        if (likely(atomic_inc_not_zero(&ce->pin_count)))
2177                return 0;
2178
2179        err = mutex_lock_interruptible(&eb->i915->drm.struct_mutex);
2180        if (err)
2181                return err;
2182
2183        err = __intel_context_do_pin(ce);
2184        mutex_unlock(&eb->i915->drm.struct_mutex);
2185
2186        return err;
2187}
2188
2189static void
2190__eb_unpin_context(struct i915_execbuffer *eb, struct intel_context *ce)
2191{
2192        if (likely(atomic_add_unless(&ce->pin_count, -1, 1)))
2193                return;
2194
2195        mutex_lock(&eb->i915->drm.struct_mutex);
2196        intel_context_unpin(ce);
2197        mutex_unlock(&eb->i915->drm.struct_mutex);
2198}
2199
2200static int __eb_pin_engine(struct i915_execbuffer *eb, struct intel_context *ce)
2201{
2202        struct intel_timeline *tl;
2203        struct i915_request *rq;
2204        int err;
2205
2206        /*
2207         * ABI: Before userspace accesses the GPU (e.g. execbuffer), report
2208         * EIO if the GPU is already wedged.
2209         */
2210        err = intel_gt_terminally_wedged(ce->engine->gt);
2211        if (err)
2212                return err;
2213
2214        /*
2215         * Pinning the contexts may generate requests in order to acquire
2216         * GGTT space, so do this first before we reserve a seqno for
2217         * ourselves.
2218         */
2219        err = __eb_pin_context(eb, ce);
2220        if (err)
2221                return err;
2222
2223        /*
2224         * Take a local wakeref for preparing to dispatch the execbuf as
2225         * we expect to access the hardware fairly frequently in the
2226         * process, and require the engine to be kept awake between accesses.
2227         * Upon dispatch, we acquire another prolonged wakeref that we hold
2228         * until the timeline is idle, which in turn releases the wakeref
2229         * taken on the engine, and the parent device.
2230         */
2231        tl = intel_context_timeline_lock(ce);
2232        if (IS_ERR(tl)) {
2233                err = PTR_ERR(tl);
2234                goto err_unpin;
2235        }
2236
2237        intel_context_enter(ce);
2238        rq = eb_throttle(ce);
2239
2240        intel_context_timeline_unlock(tl);
2241
2242        if (rq) {
2243                if (i915_request_wait(rq,
2244                                      I915_WAIT_INTERRUPTIBLE,
2245                                      MAX_SCHEDULE_TIMEOUT) < 0) {
2246                        i915_request_put(rq);
2247                        err = -EINTR;
2248                        goto err_exit;
2249                }
2250
2251                i915_request_put(rq);
2252        }
2253
2254        eb->engine = ce->engine;
2255        eb->context = ce;
2256        return 0;
2257
2258err_exit:
2259        mutex_lock(&tl->mutex);
2260        intel_context_exit(ce);
2261        intel_context_timeline_unlock(tl);
2262err_unpin:
2263        __eb_unpin_context(eb, ce);
2264        return err;
2265}
2266
2267static void eb_unpin_engine(struct i915_execbuffer *eb)
2268{
2269        struct intel_context *ce = eb->context;
2270        struct intel_timeline *tl = ce->timeline;
2271
2272        mutex_lock(&tl->mutex);
2273        intel_context_exit(ce);
2274        mutex_unlock(&tl->mutex);
2275
2276        __eb_unpin_context(eb, ce);
2277}
2278
2279static unsigned int
2280eb_select_legacy_ring(struct i915_execbuffer *eb,
2281                      struct drm_file *file,
2282                      struct drm_i915_gem_execbuffer2 *args)
2283{
2284        struct drm_i915_private *i915 = eb->i915;
2285        unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK;
2286
2287        if (user_ring_id != I915_EXEC_BSD &&
2288            (args->flags & I915_EXEC_BSD_MASK)) {
2289                DRM_DEBUG("execbuf with non bsd ring but with invalid "
2290                          "bsd dispatch flags: %d\n", (int)(args->flags));
2291                return -1;
2292        }
2293
2294        if (user_ring_id == I915_EXEC_BSD && num_vcs_engines(i915) > 1) {
2295                unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK;
2296
2297                if (bsd_idx == I915_EXEC_BSD_DEFAULT) {
2298                        bsd_idx = gen8_dispatch_bsd_engine(i915, file);
2299                } else if (bsd_idx >= I915_EXEC_BSD_RING1 &&
2300                           bsd_idx <= I915_EXEC_BSD_RING2) {
2301                        bsd_idx >>= I915_EXEC_BSD_SHIFT;
2302                        bsd_idx--;
2303                } else {
2304                        DRM_DEBUG("execbuf with unknown bsd ring: %u\n",
2305                                  bsd_idx);
2306                        return -1;
2307                }
2308
2309                return _VCS(bsd_idx);
2310        }
2311
2312        if (user_ring_id >= ARRAY_SIZE(user_ring_map)) {
2313                DRM_DEBUG("execbuf with unknown ring: %u\n", user_ring_id);
2314                return -1;
2315        }
2316
2317        return user_ring_map[user_ring_id];
2318}
2319
2320static int
2321eb_pin_engine(struct i915_execbuffer *eb,
2322              struct drm_file *file,
2323              struct drm_i915_gem_execbuffer2 *args)
2324{
2325        struct intel_context *ce;
2326        unsigned int idx;
2327        int err;
2328
2329        if (i915_gem_context_user_engines(eb->gem_context))
2330                idx = args->flags & I915_EXEC_RING_MASK;
2331        else
2332                idx = eb_select_legacy_ring(eb, file, args);
2333
2334        ce = i915_gem_context_get_engine(eb->gem_context, idx);
2335        if (IS_ERR(ce))
2336                return PTR_ERR(ce);
2337
2338        err = __eb_pin_engine(eb, ce);
2339        intel_context_put(ce);
2340
2341        return err;
2342}
2343
2344static void
2345__free_fence_array(struct drm_syncobj **fences, unsigned int n)
2346{
2347        while (n--)
2348                drm_syncobj_put(ptr_mask_bits(fences[n], 2));
2349        kvfree(fences);
2350}
2351
2352static struct drm_syncobj **
2353get_fence_array(struct drm_i915_gem_execbuffer2 *args,
2354                struct drm_file *file)
2355{
2356        const unsigned long nfences = args->num_cliprects;
2357        struct drm_i915_gem_exec_fence __user *user;
2358        struct drm_syncobj **fences;
2359        unsigned long n;
2360        int err;
2361
2362        if (!(args->flags & I915_EXEC_FENCE_ARRAY))
2363                return NULL;
2364
2365        /* Check multiplication overflow for access_ok() and kvmalloc_array() */
2366        BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long));
2367        if (nfences > min_t(unsigned long,
2368                            ULONG_MAX / sizeof(*user),
2369                            SIZE_MAX / sizeof(*fences)))
2370                return ERR_PTR(-EINVAL);
2371
2372        user = u64_to_user_ptr(args->cliprects_ptr);
2373        if (!access_ok(user, nfences * sizeof(*user)))
2374                return ERR_PTR(-EFAULT);
2375
2376        fences = kvmalloc_array(nfences, sizeof(*fences),
2377                                __GFP_NOWARN | GFP_KERNEL);
2378        if (!fences)
2379                return ERR_PTR(-ENOMEM);
2380
2381        for (n = 0; n < nfences; n++) {
2382                struct drm_i915_gem_exec_fence fence;
2383                struct drm_syncobj *syncobj;
2384
2385                if (__copy_from_user(&fence, user++, sizeof(fence))) {
2386                        err = -EFAULT;
2387                        goto err;
2388                }
2389
2390                if (fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS) {
2391                        err = -EINVAL;
2392                        goto err;
2393                }
2394
2395                syncobj = drm_syncobj_find(file, fence.handle);
2396                if (!syncobj) {
2397                        DRM_DEBUG("Invalid syncobj handle provided\n");
2398                        err = -ENOENT;
2399                        goto err;
2400                }
2401
2402                BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) &
2403                             ~__I915_EXEC_FENCE_UNKNOWN_FLAGS);
2404
2405                fences[n] = ptr_pack_bits(syncobj, fence.flags, 2);
2406        }
2407
2408        return fences;
2409
2410err:
2411        __free_fence_array(fences, n);
2412        return ERR_PTR(err);
2413}
2414
2415static void
2416put_fence_array(struct drm_i915_gem_execbuffer2 *args,
2417                struct drm_syncobj **fences)
2418{
2419        if (fences)
2420                __free_fence_array(fences, args->num_cliprects);
2421}
2422
2423static int
2424await_fence_array(struct i915_execbuffer *eb,
2425                  struct drm_syncobj **fences)
2426{
2427        const unsigned int nfences = eb->args->num_cliprects;
2428        unsigned int n;
2429        int err;
2430
2431        for (n = 0; n < nfences; n++) {
2432                struct drm_syncobj *syncobj;
2433                struct dma_fence *fence;
2434                unsigned int flags;
2435
2436                syncobj = ptr_unpack_bits(fences[n], &flags, 2);
2437                if (!(flags & I915_EXEC_FENCE_WAIT))
2438                        continue;
2439
2440                fence = drm_syncobj_fence_get(syncobj);
2441                if (!fence)
2442                        return -EINVAL;
2443
2444                err = i915_request_await_dma_fence(eb->request, fence);
2445                dma_fence_put(fence);
2446                if (err < 0)
2447                        return err;
2448        }
2449
2450        return 0;
2451}
2452
2453static void
2454signal_fence_array(struct i915_execbuffer *eb,
2455                   struct drm_syncobj **fences)
2456{
2457        const unsigned int nfences = eb->args->num_cliprects;
2458        struct dma_fence * const fence = &eb->request->fence;
2459        unsigned int n;
2460
2461        for (n = 0; n < nfences; n++) {
2462                struct drm_syncobj *syncobj;
2463                unsigned int flags;
2464
2465                syncobj = ptr_unpack_bits(fences[n], &flags, 2);
2466                if (!(flags & I915_EXEC_FENCE_SIGNAL))
2467                        continue;
2468
2469                drm_syncobj_replace_fence(syncobj, fence);
2470        }
2471}
2472
2473static int
2474i915_gem_do_execbuffer(struct drm_device *dev,
2475                       struct drm_file *file,
2476                       struct drm_i915_gem_execbuffer2 *args,
2477                       struct drm_i915_gem_exec_object2 *exec,
2478                       struct drm_syncobj **fences)
2479{
2480        struct drm_i915_private *i915 = to_i915(dev);
2481        struct i915_execbuffer eb;
2482        struct dma_fence *in_fence = NULL;
2483        struct dma_fence *exec_fence = NULL;
2484        struct sync_file *out_fence = NULL;
2485        int out_fence_fd = -1;
2486        int err;
2487
2488        BUILD_BUG_ON(__EXEC_INTERNAL_FLAGS & ~__I915_EXEC_ILLEGAL_FLAGS);
2489        BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS &
2490                     ~__EXEC_OBJECT_UNKNOWN_FLAGS);
2491
2492        eb.i915 = i915;
2493        eb.file = file;
2494        eb.args = args;
2495        if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC))
2496                args->flags |= __EXEC_HAS_RELOC;
2497
2498        eb.exec = exec;
2499        eb.vma = (struct i915_vma **)(exec + args->buffer_count + 1);
2500        eb.vma[0] = NULL;
2501        eb.flags = (unsigned int *)(eb.vma + args->buffer_count + 1);
2502
2503        eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
2504        reloc_cache_init(&eb.reloc_cache, eb.i915);
2505
2506        eb.buffer_count = args->buffer_count;
2507        eb.batch_start_offset = args->batch_start_offset;
2508        eb.batch_len = args->batch_len;
2509
2510        eb.batch_flags = 0;
2511        if (args->flags & I915_EXEC_SECURE) {
2512                if (INTEL_GEN(i915) >= 11)
2513                        return -ENODEV;
2514
2515                /* Return -EPERM to trigger fallback code on old binaries. */
2516                if (!HAS_SECURE_BATCHES(i915))
2517                        return -EPERM;
2518
2519                if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
2520                        return -EPERM;
2521
2522                eb.batch_flags |= I915_DISPATCH_SECURE;
2523        }
2524        if (args->flags & I915_EXEC_IS_PINNED)
2525                eb.batch_flags |= I915_DISPATCH_PINNED;
2526
2527        if (args->flags & I915_EXEC_FENCE_IN) {
2528                in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2));
2529                if (!in_fence)
2530                        return -EINVAL;
2531        }
2532
2533        if (args->flags & I915_EXEC_FENCE_SUBMIT) {
2534                if (in_fence) {
2535                        err = -EINVAL;
2536                        goto err_in_fence;
2537                }
2538
2539                exec_fence = sync_file_get_fence(lower_32_bits(args->rsvd2));
2540                if (!exec_fence) {
2541                        err = -EINVAL;
2542                        goto err_in_fence;
2543                }
2544        }
2545
2546        if (args->flags & I915_EXEC_FENCE_OUT) {
2547                out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
2548                if (out_fence_fd < 0) {
2549                        err = out_fence_fd;
2550                        goto err_exec_fence;
2551                }
2552        }
2553
2554        err = eb_create(&eb);
2555        if (err)
2556                goto err_out_fence;
2557
2558        GEM_BUG_ON(!eb.lut_size);
2559
2560        err = eb_select_context(&eb);
2561        if (unlikely(err))
2562                goto err_destroy;
2563
2564        err = eb_pin_engine(&eb, file, args);
2565        if (unlikely(err))
2566                goto err_context;
2567
2568        err = i915_mutex_lock_interruptible(dev);
2569        if (err)
2570                goto err_engine;
2571
2572        err = eb_relocate(&eb);
2573        if (err) {
2574                /*
2575                 * If the user expects the execobject.offset and
2576                 * reloc.presumed_offset to be an exact match,
2577                 * as for using NO_RELOC, then we cannot update
2578                 * the execobject.offset until we have completed
2579                 * relocation.
2580                 */
2581                args->flags &= ~__EXEC_HAS_RELOC;
2582                goto err_vma;
2583        }
2584
2585        if (unlikely(*eb.batch->exec_flags & EXEC_OBJECT_WRITE)) {
2586                DRM_DEBUG("Attempting to use self-modifying batch buffer\n");
2587                err = -EINVAL;
2588                goto err_vma;
2589        }
2590        if (eb.batch_start_offset > eb.batch->size ||
2591            eb.batch_len > eb.batch->size - eb.batch_start_offset) {
2592                DRM_DEBUG("Attempting to use out-of-bounds batch\n");
2593                err = -EINVAL;
2594                goto err_vma;
2595        }
2596
2597        if (eb.batch_len == 0)
2598                eb.batch_len = eb.batch->size - eb.batch_start_offset;
2599
2600        if (eb_use_cmdparser(&eb)) {
2601                struct i915_vma *vma;
2602
2603                vma = eb_parse(&eb);
2604                if (IS_ERR(vma)) {
2605                        err = PTR_ERR(vma);
2606                        goto err_vma;
2607                }
2608        }
2609
2610        /*
2611         * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
2612         * batch" bit. Hence we need to pin secure batches into the global gtt.
2613         * hsw should have this fixed, but bdw mucks it up again. */
2614        if (eb.batch_flags & I915_DISPATCH_SECURE) {
2615                struct i915_vma *vma;
2616
2617                /*
2618                 * So on first glance it looks freaky that we pin the batch here
2619                 * outside of the reservation loop. But:
2620                 * - The batch is already pinned into the relevant ppgtt, so we
2621                 *   already have the backing storage fully allocated.
2622                 * - No other BO uses the global gtt (well contexts, but meh),
2623                 *   so we don't really have issues with multiple objects not
2624                 *   fitting due to fragmentation.
2625                 * So this is actually safe.
2626                 */
2627                vma = i915_gem_object_ggtt_pin(eb.batch->obj, NULL, 0, 0, 0);
2628                if (IS_ERR(vma)) {
2629                        err = PTR_ERR(vma);
2630                        goto err_vma;
2631                }
2632
2633                eb.batch = vma;
2634        }
2635
2636        /* All GPU relocation batches must be submitted prior to the user rq */
2637        GEM_BUG_ON(eb.reloc_cache.rq);
2638
2639        /* Allocate a request for this batch buffer nice and early. */
2640        eb.request = i915_request_create(eb.context);
2641        if (IS_ERR(eb.request)) {
2642                err = PTR_ERR(eb.request);
2643                goto err_batch_unpin;
2644        }
2645
2646        if (in_fence) {
2647                err = i915_request_await_dma_fence(eb.request, in_fence);
2648                if (err < 0)
2649                        goto err_request;
2650        }
2651
2652        if (exec_fence) {
2653                err = i915_request_await_execution(eb.request, exec_fence,
2654                                                   eb.engine->bond_execute);
2655                if (err < 0)
2656                        goto err_request;
2657        }
2658
2659        if (fences) {
2660                err = await_fence_array(&eb, fences);
2661                if (err)
2662                        goto err_request;
2663        }
2664
2665        if (out_fence_fd != -1) {
2666                out_fence = sync_file_create(&eb.request->fence);
2667                if (!out_fence) {
2668                        err = -ENOMEM;
2669                        goto err_request;
2670                }
2671        }
2672
2673        /*
2674         * Whilst this request exists, batch_obj will be on the
2675         * active_list, and so will hold the active reference. Only when this
2676         * request is retired will the the batch_obj be moved onto the
2677         * inactive_list and lose its active reference. Hence we do not need
2678         * to explicitly hold another reference here.
2679         */
2680        eb.request->batch = eb.batch;
2681        if (eb.batch->private)
2682                intel_engine_pool_mark_active(eb.batch->private, eb.request);
2683
2684        trace_i915_request_queue(eb.request, eb.batch_flags);
2685        err = eb_submit(&eb);
2686err_request:
2687        add_to_client(eb.request, file);
2688        i915_request_add(eb.request);
2689
2690        if (fences)
2691                signal_fence_array(&eb, fences);
2692
2693        if (out_fence) {
2694                if (err == 0) {
2695                        fd_install(out_fence_fd, out_fence->file);
2696                        args->rsvd2 &= GENMASK_ULL(31, 0); /* keep in-fence */
2697                        args->rsvd2 |= (u64)out_fence_fd << 32;
2698                        out_fence_fd = -1;
2699                } else {
2700                        fput(out_fence->file);
2701                }
2702        }
2703
2704err_batch_unpin:
2705        if (eb.batch_flags & I915_DISPATCH_SECURE)
2706                i915_vma_unpin(eb.batch);
2707        if (eb.batch->private)
2708                intel_engine_pool_put(eb.batch->private);
2709err_vma:
2710        if (eb.exec)
2711                eb_release_vmas(&eb);
2712        mutex_unlock(&dev->struct_mutex);
2713err_engine:
2714        eb_unpin_engine(&eb);
2715err_context:
2716        i915_gem_context_put(eb.gem_context);
2717err_destroy:
2718        eb_destroy(&eb);
2719err_out_fence:
2720        if (out_fence_fd != -1)
2721                put_unused_fd(out_fence_fd);
2722err_exec_fence:
2723        dma_fence_put(exec_fence);
2724err_in_fence:
2725        dma_fence_put(in_fence);
2726        return err;
2727}
2728
2729static size_t eb_element_size(void)
2730{
2731        return (sizeof(struct drm_i915_gem_exec_object2) +
2732                sizeof(struct i915_vma *) +
2733                sizeof(unsigned int));
2734}
2735
2736static bool check_buffer_count(size_t count)
2737{
2738        const size_t sz = eb_element_size();
2739
2740        /*
2741         * When using LUT_HANDLE, we impose a limit of INT_MAX for the lookup
2742         * array size (see eb_create()). Otherwise, we can accept an array as
2743         * large as can be addressed (though use large arrays at your peril)!
2744         */
2745
2746        return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1);
2747}
2748
2749/*
2750 * Legacy execbuffer just creates an exec2 list from the original exec object
2751 * list array and passes it to the real function.
2752 */
2753int
2754i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data,
2755                          struct drm_file *file)
2756{
2757        struct drm_i915_gem_execbuffer *args = data;
2758        struct drm_i915_gem_execbuffer2 exec2;
2759        struct drm_i915_gem_exec_object *exec_list = NULL;
2760        struct drm_i915_gem_exec_object2 *exec2_list = NULL;
2761        const size_t count = args->buffer_count;
2762        unsigned int i;
2763        int err;
2764
2765        if (!check_buffer_count(count)) {
2766                DRM_DEBUG("execbuf2 with %zd buffers\n", count);
2767                return -EINVAL;
2768        }
2769
2770        exec2.buffers_ptr = args->buffers_ptr;
2771        exec2.buffer_count = args->buffer_count;
2772        exec2.batch_start_offset = args->batch_start_offset;
2773        exec2.batch_len = args->batch_len;
2774        exec2.DR1 = args->DR1;
2775        exec2.DR4 = args->DR4;
2776        exec2.num_cliprects = args->num_cliprects;
2777        exec2.cliprects_ptr = args->cliprects_ptr;
2778        exec2.flags = I915_EXEC_RENDER;
2779        i915_execbuffer2_set_context_id(exec2, 0);
2780
2781        if (!i915_gem_check_execbuffer(&exec2))
2782                return -EINVAL;
2783
2784        /* Copy in the exec list from userland */
2785        exec_list = kvmalloc_array(count, sizeof(*exec_list),
2786                                   __GFP_NOWARN | GFP_KERNEL);
2787        exec2_list = kvmalloc_array(count + 1, eb_element_size(),
2788                                    __GFP_NOWARN | GFP_KERNEL);
2789        if (exec_list == NULL || exec2_list == NULL) {
2790                DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
2791                          args->buffer_count);
2792                kvfree(exec_list);
2793                kvfree(exec2_list);
2794                return -ENOMEM;
2795        }
2796        err = copy_from_user(exec_list,
2797                             u64_to_user_ptr(args->buffers_ptr),
2798                             sizeof(*exec_list) * count);
2799        if (err) {
2800                DRM_DEBUG("copy %d exec entries failed %d\n",
2801                          args->buffer_count, err);
2802                kvfree(exec_list);
2803                kvfree(exec2_list);
2804                return -EFAULT;
2805        }
2806
2807        for (i = 0; i < args->buffer_count; i++) {
2808                exec2_list[i].handle = exec_list[i].handle;
2809                exec2_list[i].relocation_count = exec_list[i].relocation_count;
2810                exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
2811                exec2_list[i].alignment = exec_list[i].alignment;
2812                exec2_list[i].offset = exec_list[i].offset;
2813                if (INTEL_GEN(to_i915(dev)) < 4)
2814                        exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
2815                else
2816                        exec2_list[i].flags = 0;
2817        }
2818
2819        err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list, NULL);
2820        if (exec2.flags & __EXEC_HAS_RELOC) {
2821                struct drm_i915_gem_exec_object __user *user_exec_list =
2822                        u64_to_user_ptr(args->buffers_ptr);
2823
2824                /* Copy the new buffer offsets back to the user's exec list. */
2825                for (i = 0; i < args->buffer_count; i++) {
2826                        if (!(exec2_list[i].offset & UPDATE))
2827                                continue;
2828
2829                        exec2_list[i].offset =
2830                                gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
2831                        exec2_list[i].offset &= PIN_OFFSET_MASK;
2832                        if (__copy_to_user(&user_exec_list[i].offset,
2833                                           &exec2_list[i].offset,
2834                                           sizeof(user_exec_list[i].offset)))
2835                                break;
2836                }
2837        }
2838
2839        kvfree(exec_list);
2840        kvfree(exec2_list);
2841        return err;
2842}
2843
2844int
2845i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data,
2846                           struct drm_file *file)
2847{
2848        struct drm_i915_gem_execbuffer2 *args = data;
2849        struct drm_i915_gem_exec_object2 *exec2_list;
2850        struct drm_syncobj **fences = NULL;
2851        const size_t count = args->buffer_count;
2852        int err;
2853
2854        if (!check_buffer_count(count)) {
2855                DRM_DEBUG("execbuf2 with %zd buffers\n", count);
2856                return -EINVAL;
2857        }
2858
2859        if (!i915_gem_check_execbuffer(args))
2860                return -EINVAL;
2861
2862        /* Allocate an extra slot for use by the command parser */
2863        exec2_list = kvmalloc_array(count + 1, eb_element_size(),
2864                                    __GFP_NOWARN | GFP_KERNEL);
2865        if (exec2_list == NULL) {
2866                DRM_DEBUG("Failed to allocate exec list for %zd buffers\n",
2867                          count);
2868                return -ENOMEM;
2869        }
2870        if (copy_from_user(exec2_list,
2871                           u64_to_user_ptr(args->buffers_ptr),
2872                           sizeof(*exec2_list) * count)) {
2873                DRM_DEBUG("copy %zd exec entries failed\n", count);
2874                kvfree(exec2_list);
2875                return -EFAULT;
2876        }
2877
2878        if (args->flags & I915_EXEC_FENCE_ARRAY) {
2879                fences = get_fence_array(args, file);
2880                if (IS_ERR(fences)) {
2881                        kvfree(exec2_list);
2882                        return PTR_ERR(fences);
2883                }
2884        }
2885
2886        err = i915_gem_do_execbuffer(dev, file, args, exec2_list, fences);
2887
2888        /*
2889         * Now that we have begun execution of the batchbuffer, we ignore
2890         * any new error after this point. Also given that we have already
2891         * updated the associated relocations, we try to write out the current
2892         * object locations irrespective of any error.
2893         */
2894        if (args->flags & __EXEC_HAS_RELOC) {
2895                struct drm_i915_gem_exec_object2 __user *user_exec_list =
2896                        u64_to_user_ptr(args->buffers_ptr);
2897                unsigned int i;
2898
2899                /* Copy the new buffer offsets back to the user's exec list. */
2900                /*
2901                 * Note: count * sizeof(*user_exec_list) does not overflow,
2902                 * because we checked 'count' in check_buffer_count().
2903                 *
2904                 * And this range already got effectively checked earlier
2905                 * when we did the "copy_from_user()" above.
2906                 */
2907                if (!user_access_begin(user_exec_list, count * sizeof(*user_exec_list)))
2908                        goto end;
2909
2910                for (i = 0; i < args->buffer_count; i++) {
2911                        if (!(exec2_list[i].offset & UPDATE))
2912                                continue;
2913
2914                        exec2_list[i].offset =
2915                                gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
2916                        unsafe_put_user(exec2_list[i].offset,
2917                                        &user_exec_list[i].offset,
2918                                        end_user);
2919                }
2920end_user:
2921                user_access_end();
2922end:;
2923        }
2924
2925        args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS;
2926        put_fence_array(args, fences);
2927        kvfree(exec2_list);
2928        return err;
2929}
2930