1/************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27/* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 */ 30 31#ifndef _TTM_BO_API_H_ 32#define _TTM_BO_API_H_ 33 34#include "drm_hashtab.h" 35#include <linux/kref.h> 36#include <linux/list.h> 37#include <linux/wait.h> 38#include <linux/mutex.h> 39#include <linux/mm.h> 40#include <linux/rbtree.h> 41#include <linux/bitmap.h> 42 43struct ttm_bo_device; 44 45struct drm_mm_node; 46 47/** 48 * struct ttm_mem_reg 49 * 50 * @mm_node: Memory manager node. 51 * @size: Requested size of memory region. 52 * @num_pages: Actual size of memory region in pages. 53 * @page_alignment: Page alignment. 54 * @placement: Placement flags. 55 * 56 * Structure indicating the placement and space resources used by a 57 * buffer object. 58 */ 59 60struct ttm_mem_reg { 61 struct drm_mm_node *mm_node; 62 unsigned long size; 63 unsigned long num_pages; 64 uint32_t page_alignment; 65 uint32_t mem_type; 66 uint32_t placement; 67}; 68 69/** 70 * enum ttm_bo_type 71 * 72 * @ttm_bo_type_device: These are 'normal' buffers that can 73 * be mmapped by user space. Each of these bos occupy a slot in the 74 * device address space, that can be used for normal vm operations. 75 * 76 * @ttm_bo_type_user: These are user-space memory areas that are made 77 * available to the GPU by mapping the buffer pages into the GPU aperture 78 * space. These buffers cannot be mmaped from the device address space. 79 * 80 * @ttm_bo_type_kernel: These buffers are like ttm_bo_type_device buffers, 81 * but they cannot be accessed from user-space. For kernel-only use. 82 */ 83 84enum ttm_bo_type { 85 ttm_bo_type_device, 86 ttm_bo_type_user, 87 ttm_bo_type_kernel 88}; 89 90struct ttm_tt; 91 92/** 93 * struct ttm_buffer_object 94 * 95 * @bdev: Pointer to the buffer object device structure. 96 * @buffer_start: The virtual user-space start address of ttm_bo_type_user 97 * buffers. 98 * @type: The bo type. 99 * @destroy: Destruction function. If NULL, kfree is used. 100 * @num_pages: Actual number of pages. 101 * @addr_space_offset: Address space offset. 102 * @acc_size: Accounted size for this object. 103 * @kref: Reference count of this buffer object. When this refcount reaches 104 * zero, the object is put on the delayed delete list. 105 * @list_kref: List reference count of this buffer object. This member is 106 * used to avoid destruction while the buffer object is still on a list. 107 * Lru lists may keep one refcount, the delayed delete list, and kref != 0 108 * keeps one refcount. When this refcount reaches zero, 109 * the object is destroyed. 110 * @event_queue: Queue for processes waiting on buffer object status change. 111 * @lock: spinlock protecting mostly synchronization members. 112 * @proposed_placement: Proposed placement for the buffer. Changed only by the 113 * creator prior to validation as opposed to bo->mem.proposed_flags which is 114 * changed by the implementation prior to a buffer move if it wants to outsmart 115 * the buffer creator / user. This latter happens, for example, at eviction. 116 * @mem: structure describing current placement. 117 * @persistant_swap_storage: Usually the swap storage is deleted for buffers 118 * pinned in physical memory. If this behaviour is not desired, this member 119 * holds a pointer to a persistant shmem object. 120 * @ttm: TTM structure holding system pages. 121 * @evicted: Whether the object was evicted without user-space knowing. 122 * @cpu_writes: For synchronization. Number of cpu writers. 123 * @lru: List head for the lru list. 124 * @ddestroy: List head for the delayed destroy list. 125 * @swap: List head for swap LRU list. 126 * @val_seq: Sequence of the validation holding the @reserved lock. 127 * Used to avoid starvation when many processes compete to validate the 128 * buffer. This member is protected by the bo_device::lru_lock. 129 * @seq_valid: The value of @val_seq is valid. This value is protected by 130 * the bo_device::lru_lock. 131 * @reserved: Deadlock-free lock used for synchronization state transitions. 132 * @sync_obj_arg: Opaque argument to synchronization object function. 133 * @sync_obj: Pointer to a synchronization object. 134 * @priv_flags: Flags describing buffer object internal state. 135 * @vm_rb: Rb node for the vm rb tree. 136 * @vm_node: Address space manager node. 137 * @offset: The current GPU offset, which can have different meanings 138 * depending on the memory type. For SYSTEM type memory, it should be 0. 139 * @cur_placement: Hint of current placement. 140 * 141 * Base class for TTM buffer object, that deals with data placement and CPU 142 * mappings. GPU mappings are really up to the driver, but for simpler GPUs 143 * the driver can usually use the placement offset @offset directly as the 144 * GPU virtual address. For drivers implementing multiple 145 * GPU memory manager contexts, the driver should manage the address space 146 * in these contexts separately and use these objects to get the correct 147 * placement and caching for these GPU maps. This makes it possible to use 148 * these objects for even quite elaborate memory management schemes. 149 * The destroy member, the API visibility of this object makes it possible 150 * to derive driver specific types. 151 */ 152 153struct ttm_buffer_object { 154 /** 155 * Members constant at init. 156 */ 157 158 struct ttm_bo_global *glob; 159 struct ttm_bo_device *bdev; 160 unsigned long buffer_start; 161 enum ttm_bo_type type; 162 void (*destroy) (struct ttm_buffer_object *); 163 unsigned long num_pages; 164 uint64_t addr_space_offset; 165 size_t acc_size; 166 167 /** 168 * Members not needing protection. 169 */ 170 171 struct kref kref; 172 struct kref list_kref; 173 wait_queue_head_t event_queue; 174 spinlock_t lock; 175 176 /** 177 * Members protected by the bo::reserved lock. 178 */ 179 180 uint32_t proposed_placement; 181 struct ttm_mem_reg mem; 182 struct file *persistant_swap_storage; 183 struct ttm_tt *ttm; 184 bool evicted; 185 186 /** 187 * Members protected by the bo::reserved lock only when written to. 188 */ 189 190 atomic_t cpu_writers; 191 192 /** 193 * Members protected by the bdev::lru_lock. 194 */ 195 196 struct list_head lru; 197 struct list_head ddestroy; 198 struct list_head swap; 199 uint32_t val_seq; 200 bool seq_valid; 201 202 /** 203 * Members protected by the bdev::lru_lock 204 * only when written to. 205 */ 206 207 atomic_t reserved; 208 209 210 /** 211 * Members protected by the bo::lock 212 */ 213 214 void *sync_obj_arg; 215 void *sync_obj; 216 unsigned long priv_flags; 217 218 /** 219 * Members protected by the bdev::vm_lock 220 */ 221 222 struct rb_node vm_rb; 223 struct drm_mm_node *vm_node; 224 225 226 /** 227 * Special members that are protected by the reserve lock 228 * and the bo::lock when written to. Can be read with 229 * either of these locks held. 230 */ 231 232 unsigned long offset; 233 uint32_t cur_placement; 234}; 235 236/** 237 * struct ttm_bo_kmap_obj 238 * 239 * @virtual: The current kernel virtual address. 240 * @page: The page when kmap'ing a single page. 241 * @bo_kmap_type: Type of bo_kmap. 242 * 243 * Object describing a kernel mapping. Since a TTM bo may be located 244 * in various memory types with various caching policies, the 245 * mapping can either be an ioremap, a vmap, a kmap or part of a 246 * premapped region. 247 */ 248 249#define TTM_BO_MAP_IOMEM_MASK 0x80 250struct ttm_bo_kmap_obj { 251 void *virtual; 252 struct page *page; 253 enum { 254 ttm_bo_map_iomap = 1 | TTM_BO_MAP_IOMEM_MASK, 255 ttm_bo_map_vmap = 2, 256 ttm_bo_map_kmap = 3, 257 ttm_bo_map_premapped = 4 | TTM_BO_MAP_IOMEM_MASK, 258 } bo_kmap_type; 259}; 260 261/** 262 * ttm_bo_reference - reference a struct ttm_buffer_object 263 * 264 * @bo: The buffer object. 265 * 266 * Returns a refcounted pointer to a buffer object. 267 */ 268 269static inline struct ttm_buffer_object * 270ttm_bo_reference(struct ttm_buffer_object *bo) 271{ 272 kref_get(&bo->kref); 273 return bo; 274} 275 276/** 277 * ttm_bo_wait - wait for buffer idle. 278 * 279 * @bo: The buffer object. 280 * @interruptible: Use interruptible wait. 281 * @no_wait: Return immediately if buffer is busy. 282 * 283 * This function must be called with the bo::mutex held, and makes 284 * sure any previous rendering to the buffer is completed. 285 * Note: It might be necessary to block validations before the 286 * wait by reserving the buffer. 287 * Returns -EBUSY if no_wait is true and the buffer is busy. 288 * Returns -ERESTART if interrupted by a signal. 289 */ 290extern int ttm_bo_wait(struct ttm_buffer_object *bo, bool lazy, 291 bool interruptible, bool no_wait); 292/** 293 * ttm_buffer_object_validate 294 * 295 * @bo: The buffer object. 296 * @proposed_placement: Proposed_placement for the buffer object. 297 * @interruptible: Sleep interruptible if sleeping. 298 * @no_wait: Return immediately if the buffer is busy. 299 * 300 * Changes placement and caching policy of the buffer object 301 * according to bo::proposed_flags. 302 * Returns 303 * -EINVAL on invalid proposed_flags. 304 * -ENOMEM on out-of-memory condition. 305 * -EBUSY if no_wait is true and buffer busy. 306 * -ERESTART if interrupted by a signal. 307 */ 308extern int ttm_buffer_object_validate(struct ttm_buffer_object *bo, 309 uint32_t proposed_placement, 310 bool interruptible, bool no_wait); 311/** 312 * ttm_bo_unref 313 * 314 * @bo: The buffer object. 315 * 316 * Unreference and clear a pointer to a buffer object. 317 */ 318extern void ttm_bo_unref(struct ttm_buffer_object **bo); 319 320/** 321 * ttm_bo_synccpu_write_grab 322 * 323 * @bo: The buffer object: 324 * @no_wait: Return immediately if buffer is busy. 325 * 326 * Synchronizes a buffer object for CPU RW access. This means 327 * blocking command submission that affects the buffer and 328 * waiting for buffer idle. This lock is recursive. 329 * Returns 330 * -EBUSY if the buffer is busy and no_wait is true. 331 * -ERESTART if interrupted by a signal. 332 */ 333 334extern int 335ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait); 336/** 337 * ttm_bo_synccpu_write_release: 338 * 339 * @bo : The buffer object. 340 * 341 * Releases a synccpu lock. 342 */ 343extern void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo); 344 345/** 346 * ttm_buffer_object_init 347 * 348 * @bdev: Pointer to a ttm_bo_device struct. 349 * @bo: Pointer to a ttm_buffer_object to be initialized. 350 * @size: Requested size of buffer object. 351 * @type: Requested type of buffer object. 352 * @flags: Initial placement flags. 353 * @page_alignment: Data alignment in pages. 354 * @buffer_start: Virtual address of user space data backing a 355 * user buffer object. 356 * @interruptible: If needing to sleep to wait for GPU resources, 357 * sleep interruptible. 358 * @persistant_swap_storage: Usually the swap storage is deleted for buffers 359 * pinned in physical memory. If this behaviour is not desired, this member 360 * holds a pointer to a persistant shmem object. Typically, this would 361 * point to the shmem object backing a GEM object if TTM is used to back a 362 * GEM user interface. 363 * @acc_size: Accounted size for this object. 364 * @destroy: Destroy function. Use NULL for kfree(). 365 * 366 * This function initializes a pre-allocated struct ttm_buffer_object. 367 * As this object may be part of a larger structure, this function, 368 * together with the @destroy function, 369 * enables driver-specific objects derived from a ttm_buffer_object. 370 * On successful return, the object kref and list_kref are set to 1. 371 * Returns 372 * -ENOMEM: Out of memory. 373 * -EINVAL: Invalid placement flags. 374 * -ERESTART: Interrupted by signal while sleeping waiting for resources. 375 */ 376 377extern int ttm_buffer_object_init(struct ttm_bo_device *bdev, 378 struct ttm_buffer_object *bo, 379 unsigned long size, 380 enum ttm_bo_type type, 381 uint32_t flags, 382 uint32_t page_alignment, 383 unsigned long buffer_start, 384 bool interrubtible, 385 struct file *persistant_swap_storage, 386 size_t acc_size, 387 void (*destroy) (struct ttm_buffer_object *)); 388/** 389 * ttm_bo_synccpu_object_init 390 * 391 * @bdev: Pointer to a ttm_bo_device struct. 392 * @bo: Pointer to a ttm_buffer_object to be initialized. 393 * @size: Requested size of buffer object. 394 * @type: Requested type of buffer object. 395 * @flags: Initial placement flags. 396 * @page_alignment: Data alignment in pages. 397 * @buffer_start: Virtual address of user space data backing a 398 * user buffer object. 399 * @interruptible: If needing to sleep while waiting for GPU resources, 400 * sleep interruptible. 401 * @persistant_swap_storage: Usually the swap storage is deleted for buffers 402 * pinned in physical memory. If this behaviour is not desired, this member 403 * holds a pointer to a persistant shmem object. Typically, this would 404 * point to the shmem object backing a GEM object if TTM is used to back a 405 * GEM user interface. 406 * @p_bo: On successful completion *p_bo points to the created object. 407 * 408 * This function allocates a ttm_buffer_object, and then calls 409 * ttm_buffer_object_init on that object. 410 * The destroy function is set to kfree(). 411 * Returns 412 * -ENOMEM: Out of memory. 413 * -EINVAL: Invalid placement flags. 414 * -ERESTART: Interrupted by signal while waiting for resources. 415 */ 416 417extern int ttm_buffer_object_create(struct ttm_bo_device *bdev, 418 unsigned long size, 419 enum ttm_bo_type type, 420 uint32_t flags, 421 uint32_t page_alignment, 422 unsigned long buffer_start, 423 bool interruptible, 424 struct file *persistant_swap_storage, 425 struct ttm_buffer_object **p_bo); 426 427/** 428 * ttm_bo_check_placement 429 * 430 * @bo: the buffer object. 431 * @set_flags: placement flags to set. 432 * @clr_flags: placement flags to clear. 433 * 434 * Performs minimal validity checking on an intended change of 435 * placement flags. 436 * Returns 437 * -EINVAL: Intended change is invalid or not allowed. 438 */ 439 440extern int ttm_bo_check_placement(struct ttm_buffer_object *bo, 441 uint32_t set_flags, uint32_t clr_flags); 442 443/** 444 * ttm_bo_init_mm 445 * 446 * @bdev: Pointer to a ttm_bo_device struct. 447 * @mem_type: The memory type. 448 * @p_offset: offset for managed area in pages. 449 * @p_size: size managed area in pages. 450 * 451 * Initialize a manager for a given memory type. 452 * Note: if part of driver firstopen, it must be protected from a 453 * potentially racing lastclose. 454 * Returns: 455 * -EINVAL: invalid size or memory type. 456 * -ENOMEM: Not enough memory. 457 * May also return driver-specified errors. 458 */ 459 460extern int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 461 unsigned long p_offset, unsigned long p_size); 462/** 463 * ttm_bo_clean_mm 464 * 465 * @bdev: Pointer to a ttm_bo_device struct. 466 * @mem_type: The memory type. 467 * 468 * Take down a manager for a given memory type after first walking 469 * the LRU list to evict any buffers left alive. 470 * 471 * Normally, this function is part of lastclose() or unload(), and at that 472 * point there shouldn't be any buffers left created by user-space, since 473 * there should've been removed by the file descriptor release() method. 474 * However, before this function is run, make sure to signal all sync objects, 475 * and verify that the delayed delete queue is empty. The driver must also 476 * make sure that there are no NO_EVICT buffers present in this memory type 477 * when the call is made. 478 * 479 * If this function is part of a VT switch, the caller must make sure that 480 * there are no appications currently validating buffers before this 481 * function is called. The caller can do that by first taking the 482 * struct ttm_bo_device::ttm_lock in write mode. 483 * 484 * Returns: 485 * -EINVAL: invalid or uninitialized memory type. 486 * -EBUSY: There are still buffers left in this memory type. 487 */ 488 489extern int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type); 490 491/** 492 * ttm_bo_evict_mm 493 * 494 * @bdev: Pointer to a ttm_bo_device struct. 495 * @mem_type: The memory type. 496 * 497 * Evicts all buffers on the lru list of the memory type. 498 * This is normally part of a VT switch or an 499 * out-of-memory-space-due-to-fragmentation handler. 500 * The caller must make sure that there are no other processes 501 * currently validating buffers, and can do that by taking the 502 * struct ttm_bo_device::ttm_lock in write mode. 503 * 504 * Returns: 505 * -EINVAL: Invalid or uninitialized memory type. 506 * -ERESTART: The call was interrupted by a signal while waiting to 507 * evict a buffer. 508 */ 509 510extern int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type); 511 512/** 513 * ttm_kmap_obj_virtual 514 * 515 * @map: A struct ttm_bo_kmap_obj returned from ttm_bo_kmap. 516 * @is_iomem: Pointer to an integer that on return indicates 1 if the 517 * virtual map is io memory, 0 if normal memory. 518 * 519 * Returns the virtual address of a buffer object area mapped by ttm_bo_kmap. 520 * If *is_iomem is 1 on return, the virtual address points to an io memory area, 521 * that should strictly be accessed by the iowriteXX() and similar functions. 522 */ 523 524static inline void *ttm_kmap_obj_virtual(struct ttm_bo_kmap_obj *map, 525 bool *is_iomem) 526{ 527 *is_iomem = !!(map->bo_kmap_type & TTM_BO_MAP_IOMEM_MASK); 528 return map->virtual; 529} 530 531/** 532 * ttm_bo_kmap 533 * 534 * @bo: The buffer object. 535 * @start_page: The first page to map. 536 * @num_pages: Number of pages to map. 537 * @map: pointer to a struct ttm_bo_kmap_obj representing the map. 538 * 539 * Sets up a kernel virtual mapping, using ioremap, vmap or kmap to the 540 * data in the buffer object. The ttm_kmap_obj_virtual function can then be 541 * used to obtain a virtual address to the data. 542 * 543 * Returns 544 * -ENOMEM: Out of memory. 545 * -EINVAL: Invalid range. 546 */ 547 548extern int ttm_bo_kmap(struct ttm_buffer_object *bo, unsigned long start_page, 549 unsigned long num_pages, struct ttm_bo_kmap_obj *map); 550 551/** 552 * ttm_bo_kunmap 553 * 554 * @map: Object describing the map to unmap. 555 * 556 * Unmaps a kernel map set up by ttm_bo_kmap. 557 */ 558 559extern void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map); 560 561#if 0 562#endif 563 564/** 565 * ttm_fbdev_mmap - mmap fbdev memory backed by a ttm buffer object. 566 * 567 * @vma: vma as input from the fbdev mmap method. 568 * @bo: The bo backing the address space. The address space will 569 * have the same size as the bo, and start at offset 0. 570 * 571 * This function is intended to be called by the fbdev mmap method 572 * if the fbdev address space is to be backed by a bo. 573 */ 574 575extern int ttm_fbdev_mmap(struct vm_area_struct *vma, 576 struct ttm_buffer_object *bo); 577 578/** 579 * ttm_bo_mmap - mmap out of the ttm device address space. 580 * 581 * @filp: filp as input from the mmap method. 582 * @vma: vma as input from the mmap method. 583 * @bdev: Pointer to the ttm_bo_device with the address space manager. 584 * 585 * This function is intended to be called by the device mmap method. 586 * if the device address space is to be backed by the bo manager. 587 */ 588 589extern int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, 590 struct ttm_bo_device *bdev); 591 592/** 593 * ttm_bo_io 594 * 595 * @bdev: Pointer to the struct ttm_bo_device. 596 * @filp: Pointer to the struct file attempting to read / write. 597 * @wbuf: User-space pointer to address of buffer to write. NULL on read. 598 * @rbuf: User-space pointer to address of buffer to read into. 599 * Null on write. 600 * @count: Number of bytes to read / write. 601 * @f_pos: Pointer to current file position. 602 * @write: 1 for read, 0 for write. 603 * 604 * This function implements read / write into ttm buffer objects, and is 605 * intended to 606 * be called from the fops::read and fops::write method. 607 * Returns: 608 * See man (2) write, man(2) read. In particular, 609 * the function may return -EINTR if 610 * interrupted by a signal. 611 */ 612 613extern ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp, 614 const char __user *wbuf, char __user *rbuf, 615 size_t count, loff_t *f_pos, bool write); 616 617extern void ttm_bo_swapout_all(struct ttm_bo_device *bdev); 618 619#endif 620