1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Fence mechanism for dma-buf to allow for asynchronous dma access 4 * 5 * Copyright (C) 2012 Canonical Ltd 6 * Copyright (C) 2012 Texas Instruments 7 * 8 * Authors: 9 * Rob Clark <robdclark@gmail.com> 10 * Maarten Lankhorst <maarten.lankhorst@canonical.com> 11 */ 12 13#ifndef __LINUX_DMA_FENCE_H 14#define __LINUX_DMA_FENCE_H 15 16#include <linux/err.h> 17#include <linux/wait.h> 18#include <linux/list.h> 19#include <linux/bitops.h> 20#include <linux/kref.h> 21#include <linux/sched.h> 22#include <linux/printk.h> 23#include <linux/rcupdate.h> 24 25struct dma_fence; 26struct dma_fence_ops; 27struct dma_fence_cb; 28 29/** 30 * struct dma_fence - software synchronization primitive 31 * @refcount: refcount for this fence 32 * @ops: dma_fence_ops associated with this fence 33 * @rcu: used for releasing fence with kfree_rcu 34 * @cb_list: list of all callbacks to call 35 * @lock: spin_lock_irqsave used for locking 36 * @context: execution context this fence belongs to, returned by 37 * dma_fence_context_alloc() 38 * @seqno: the sequence number of this fence inside the execution context, 39 * can be compared to decide which fence would be signaled later. 40 * @flags: A mask of DMA_FENCE_FLAG_* defined below 41 * @timestamp: Timestamp when the fence was signaled. 42 * @error: Optional, only valid if < 0, must be set before calling 43 * dma_fence_signal, indicates that the fence has completed with an error. 44 * 45 * the flags member must be manipulated and read using the appropriate 46 * atomic ops (bit_*), so taking the spinlock will not be needed most 47 * of the time. 48 * 49 * DMA_FENCE_FLAG_SIGNALED_BIT - fence is already signaled 50 * DMA_FENCE_FLAG_TIMESTAMP_BIT - timestamp recorded for fence signaling 51 * DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT - enable_signaling might have been called 52 * DMA_FENCE_FLAG_USER_BITS - start of the unused bits, can be used by the 53 * implementer of the fence for its own purposes. Can be used in different 54 * ways by different fence implementers, so do not rely on this. 55 * 56 * Since atomic bitops are used, this is not guaranteed to be the case. 57 * Particularly, if the bit was set, but dma_fence_signal was called right 58 * before this bit was set, it would have been able to set the 59 * DMA_FENCE_FLAG_SIGNALED_BIT, before enable_signaling was called. 60 * Adding a check for DMA_FENCE_FLAG_SIGNALED_BIT after setting 61 * DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT closes this race, and makes sure that 62 * after dma_fence_signal was called, any enable_signaling call will have either 63 * been completed, or never called at all. 64 */ 65struct dma_fence { 66 struct kref refcount; 67 const struct dma_fence_ops *ops; 68 struct rcu_head rcu; 69 struct list_head cb_list; 70 spinlock_t *lock; 71 u64 context; 72 u64 seqno; 73 unsigned long flags; 74 ktime_t timestamp; 75 int error; 76}; 77 78enum dma_fence_flag_bits { 79 DMA_FENCE_FLAG_SIGNALED_BIT, 80 DMA_FENCE_FLAG_TIMESTAMP_BIT, 81 DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, 82 DMA_FENCE_FLAG_USER_BITS, /* must always be last member */ 83}; 84 85typedef void (*dma_fence_func_t)(struct dma_fence *fence, 86 struct dma_fence_cb *cb); 87 88/** 89 * struct dma_fence_cb - callback for dma_fence_add_callback() 90 * @node: used by dma_fence_add_callback() to append this struct to fence::cb_list 91 * @func: dma_fence_func_t to call 92 * 93 * This struct will be initialized by dma_fence_add_callback(), additional 94 * data can be passed along by embedding dma_fence_cb in another struct. 95 */ 96struct dma_fence_cb { 97 struct list_head node; 98 dma_fence_func_t func; 99}; 100 101/** 102 * struct dma_fence_ops - operations implemented for fence 103 * 104 */ 105struct dma_fence_ops { 106 /** 107 * @use_64bit_seqno: 108 * 109 * True if this dma_fence implementation uses 64bit seqno, false 110 * otherwise. 111 */ 112 bool use_64bit_seqno; 113 114 /** 115 * @get_driver_name: 116 * 117 * Returns the driver name. This is a callback to allow drivers to 118 * compute the name at runtime, without having it to store permanently 119 * for each fence, or build a cache of some sort. 120 * 121 * This callback is mandatory. 122 */ 123 const char * (*get_driver_name)(struct dma_fence *fence); 124 125 /** 126 * @get_timeline_name: 127 * 128 * Return the name of the context this fence belongs to. This is a 129 * callback to allow drivers to compute the name at runtime, without 130 * having it to store permanently for each fence, or build a cache of 131 * some sort. 132 * 133 * This callback is mandatory. 134 */ 135 const char * (*get_timeline_name)(struct dma_fence *fence); 136 137 /** 138 * @enable_signaling: 139 * 140 * Enable software signaling of fence. 141 * 142 * For fence implementations that have the capability for hw->hw 143 * signaling, they can implement this op to enable the necessary 144 * interrupts, or insert commands into cmdstream, etc, to avoid these 145 * costly operations for the common case where only hw->hw 146 * synchronization is required. This is called in the first 147 * dma_fence_wait() or dma_fence_add_callback() path to let the fence 148 * implementation know that there is another driver waiting on the 149 * signal (ie. hw->sw case). 150 * 151 * This function can be called from atomic context, but not 152 * from irq context, so normal spinlocks can be used. 153 * 154 * A return value of false indicates the fence already passed, 155 * or some failure occurred that made it impossible to enable 156 * signaling. True indicates successful enabling. 157 * 158 * &dma_fence.error may be set in enable_signaling, but only when false 159 * is returned. 160 * 161 * Since many implementations can call dma_fence_signal() even when before 162 * @enable_signaling has been called there's a race window, where the 163 * dma_fence_signal() might result in the final fence reference being 164 * released and its memory freed. To avoid this, implementations of this 165 * callback should grab their own reference using dma_fence_get(), to be 166 * released when the fence is signalled (through e.g. the interrupt 167 * handler). 168 * 169 * This callback is optional. If this callback is not present, then the 170 * driver must always have signaling enabled. 171 */ 172 bool (*enable_signaling)(struct dma_fence *fence); 173 174 /** 175 * @signaled: 176 * 177 * Peek whether the fence is signaled, as a fastpath optimization for 178 * e.g. dma_fence_wait() or dma_fence_add_callback(). Note that this 179 * callback does not need to make any guarantees beyond that a fence 180 * once indicates as signalled must always return true from this 181 * callback. This callback may return false even if the fence has 182 * completed already, in this case information hasn't propogated throug 183 * the system yet. See also dma_fence_is_signaled(). 184 * 185 * May set &dma_fence.error if returning true. 186 * 187 * This callback is optional. 188 */ 189 bool (*signaled)(struct dma_fence *fence); 190 191 /** 192 * @wait: 193 * 194 * Custom wait implementation, defaults to dma_fence_default_wait() if 195 * not set. 196 * 197 * The dma_fence_default_wait implementation should work for any fence, as long 198 * as @enable_signaling works correctly. This hook allows drivers to 199 * have an optimized version for the case where a process context is 200 * already available, e.g. if @enable_signaling for the general case 201 * needs to set up a worker thread. 202 * 203 * Must return -ERESTARTSYS if the wait is intr = true and the wait was 204 * interrupted, and remaining jiffies if fence has signaled, or 0 if wait 205 * timed out. Can also return other error values on custom implementations, 206 * which should be treated as if the fence is signaled. For example a hardware 207 * lockup could be reported like that. 208 * 209 * This callback is optional. 210 */ 211 signed long (*wait)(struct dma_fence *fence, 212 bool intr, signed long timeout); 213 214 /** 215 * @release: 216 * 217 * Called on destruction of fence to release additional resources. 218 * Can be called from irq context. This callback is optional. If it is 219 * NULL, then dma_fence_free() is instead called as the default 220 * implementation. 221 */ 222 void (*release)(struct dma_fence *fence); 223 224 /** 225 * @fence_value_str: 226 * 227 * Callback to fill in free-form debug info specific to this fence, like 228 * the sequence number. 229 * 230 * This callback is optional. 231 */ 232 void (*fence_value_str)(struct dma_fence *fence, char *str, int size); 233 234 /** 235 * @timeline_value_str: 236 * 237 * Fills in the current value of the timeline as a string, like the 238 * sequence number. Note that the specific fence passed to this function 239 * should not matter, drivers should only use it to look up the 240 * corresponding timeline structures. 241 */ 242 void (*timeline_value_str)(struct dma_fence *fence, 243 char *str, int size); 244}; 245 246void dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, 247 spinlock_t *lock, u64 context, u64 seqno); 248 249void dma_fence_release(struct kref *kref); 250void dma_fence_free(struct dma_fence *fence); 251 252/** 253 * dma_fence_put - decreases refcount of the fence 254 * @fence: fence to reduce refcount of 255 */ 256static inline void dma_fence_put(struct dma_fence *fence) 257{ 258 if (fence) 259 kref_put(&fence->refcount, dma_fence_release); 260} 261 262/** 263 * dma_fence_get - increases refcount of the fence 264 * @fence: fence to increase refcount of 265 * 266 * Returns the same fence, with refcount increased by 1. 267 */ 268static inline struct dma_fence *dma_fence_get(struct dma_fence *fence) 269{ 270 if (fence) 271 kref_get(&fence->refcount); 272 return fence; 273} 274 275/** 276 * dma_fence_get_rcu - get a fence from a reservation_object_list with 277 * rcu read lock 278 * @fence: fence to increase refcount of 279 * 280 * Function returns NULL if no refcount could be obtained, or the fence. 281 */ 282static inline struct dma_fence *dma_fence_get_rcu(struct dma_fence *fence) 283{ 284 if (kref_get_unless_zero(&fence->refcount)) 285 return fence; 286 else 287 return NULL; 288} 289 290/** 291 * dma_fence_get_rcu_safe - acquire a reference to an RCU tracked fence 292 * @fencep: pointer to fence to increase refcount of 293 * 294 * Function returns NULL if no refcount could be obtained, or the fence. 295 * This function handles acquiring a reference to a fence that may be 296 * reallocated within the RCU grace period (such as with SLAB_TYPESAFE_BY_RCU), 297 * so long as the caller is using RCU on the pointer to the fence. 298 * 299 * An alternative mechanism is to employ a seqlock to protect a bunch of 300 * fences, such as used by struct reservation_object. When using a seqlock, 301 * the seqlock must be taken before and checked after a reference to the 302 * fence is acquired (as shown here). 303 * 304 * The caller is required to hold the RCU read lock. 305 */ 306static inline struct dma_fence * 307dma_fence_get_rcu_safe(struct dma_fence __rcu **fencep) 308{ 309 do { 310 struct dma_fence *fence; 311 312 fence = rcu_dereference(*fencep); 313 if (!fence) 314 return NULL; 315 316 if (!dma_fence_get_rcu(fence)) 317 continue; 318 319 /* The atomic_inc_not_zero() inside dma_fence_get_rcu() 320 * provides a full memory barrier upon success (such as now). 321 * This is paired with the write barrier from assigning 322 * to the __rcu protected fence pointer so that if that 323 * pointer still matches the current fence, we know we 324 * have successfully acquire a reference to it. If it no 325 * longer matches, we are holding a reference to some other 326 * reallocated pointer. This is possible if the allocator 327 * is using a freelist like SLAB_TYPESAFE_BY_RCU where the 328 * fence remains valid for the RCU grace period, but it 329 * may be reallocated. When using such allocators, we are 330 * responsible for ensuring the reference we get is to 331 * the right fence, as below. 332 */ 333 if (fence == rcu_access_pointer(*fencep)) 334 return rcu_pointer_handoff(fence); 335 336 dma_fence_put(fence); 337 } while (1); 338} 339 340int dma_fence_signal(struct dma_fence *fence); 341int dma_fence_signal_locked(struct dma_fence *fence); 342signed long dma_fence_default_wait(struct dma_fence *fence, 343 bool intr, signed long timeout); 344int dma_fence_add_callback(struct dma_fence *fence, 345 struct dma_fence_cb *cb, 346 dma_fence_func_t func); 347bool dma_fence_remove_callback(struct dma_fence *fence, 348 struct dma_fence_cb *cb); 349void dma_fence_enable_sw_signaling(struct dma_fence *fence); 350 351/** 352 * dma_fence_is_signaled_locked - Return an indication if the fence 353 * is signaled yet. 354 * @fence: the fence to check 355 * 356 * Returns true if the fence was already signaled, false if not. Since this 357 * function doesn't enable signaling, it is not guaranteed to ever return 358 * true if dma_fence_add_callback(), dma_fence_wait() or 359 * dma_fence_enable_sw_signaling() haven't been called before. 360 * 361 * This function requires &dma_fence.lock to be held. 362 * 363 * See also dma_fence_is_signaled(). 364 */ 365static inline bool 366dma_fence_is_signaled_locked(struct dma_fence *fence) 367{ 368 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 369 return true; 370 371 if (fence->ops->signaled && fence->ops->signaled(fence)) { 372 dma_fence_signal_locked(fence); 373 return true; 374 } 375 376 return false; 377} 378 379/** 380 * dma_fence_is_signaled - Return an indication if the fence is signaled yet. 381 * @fence: the fence to check 382 * 383 * Returns true if the fence was already signaled, false if not. Since this 384 * function doesn't enable signaling, it is not guaranteed to ever return 385 * true if dma_fence_add_callback(), dma_fence_wait() or 386 * dma_fence_enable_sw_signaling() haven't been called before. 387 * 388 * It's recommended for seqno fences to call dma_fence_signal when the 389 * operation is complete, it makes it possible to prevent issues from 390 * wraparound between time of issue and time of use by checking the return 391 * value of this function before calling hardware-specific wait instructions. 392 * 393 * See also dma_fence_is_signaled_locked(). 394 */ 395static inline bool 396dma_fence_is_signaled(struct dma_fence *fence) 397{ 398 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) 399 return true; 400 401 if (fence->ops->signaled && fence->ops->signaled(fence)) { 402 dma_fence_signal(fence); 403 return true; 404 } 405 406 return false; 407} 408 409/** 410 * __dma_fence_is_later - return if f1 is chronologically later than f2 411 * @f1: the first fence's seqno 412 * @f2: the second fence's seqno from the same context 413 * @ops: dma_fence_ops associated with the seqno 414 * 415 * Returns true if f1 is chronologically later than f2. Both fences must be 416 * from the same context, since a seqno is not common across contexts. 417 */ 418static inline bool __dma_fence_is_later(u64 f1, u64 f2, 419 const struct dma_fence_ops *ops) 420{ 421 /* This is for backward compatibility with drivers which can only handle 422 * 32bit sequence numbers. Use a 64bit compare when the driver says to 423 * do so. 424 */ 425 if (ops->use_64bit_seqno) 426 return f1 > f2; 427 428 return (int)(lower_32_bits(f1) - lower_32_bits(f2)) > 0; 429} 430 431/** 432 * dma_fence_is_later - return if f1 is chronologically later than f2 433 * @f1: the first fence from the same context 434 * @f2: the second fence from the same context 435 * 436 * Returns true if f1 is chronologically later than f2. Both fences must be 437 * from the same context, since a seqno is not re-used across contexts. 438 */ 439static inline bool dma_fence_is_later(struct dma_fence *f1, 440 struct dma_fence *f2) 441{ 442 if (WARN_ON(f1->context != f2->context)) 443 return false; 444 445 return __dma_fence_is_later(f1->seqno, f2->seqno, f1->ops); 446} 447 448/** 449 * dma_fence_later - return the chronologically later fence 450 * @f1: the first fence from the same context 451 * @f2: the second fence from the same context 452 * 453 * Returns NULL if both fences are signaled, otherwise the fence that would be 454 * signaled last. Both fences must be from the same context, since a seqno is 455 * not re-used across contexts. 456 */ 457static inline struct dma_fence *dma_fence_later(struct dma_fence *f1, 458 struct dma_fence *f2) 459{ 460 if (WARN_ON(f1->context != f2->context)) 461 return NULL; 462 463 /* 464 * Can't check just DMA_FENCE_FLAG_SIGNALED_BIT here, it may never 465 * have been set if enable_signaling wasn't called, and enabling that 466 * here is overkill. 467 */ 468 if (dma_fence_is_later(f1, f2)) 469 return dma_fence_is_signaled(f1) ? NULL : f1; 470 else 471 return dma_fence_is_signaled(f2) ? NULL : f2; 472} 473 474/** 475 * dma_fence_get_status_locked - returns the status upon completion 476 * @fence: the dma_fence to query 477 * 478 * Drivers can supply an optional error status condition before they signal 479 * the fence (to indicate whether the fence was completed due to an error 480 * rather than success). The value of the status condition is only valid 481 * if the fence has been signaled, dma_fence_get_status_locked() first checks 482 * the signal state before reporting the error status. 483 * 484 * Returns 0 if the fence has not yet been signaled, 1 if the fence has 485 * been signaled without an error condition, or a negative error code 486 * if the fence has been completed in err. 487 */ 488static inline int dma_fence_get_status_locked(struct dma_fence *fence) 489{ 490 if (dma_fence_is_signaled_locked(fence)) 491 return fence->error ?: 1; 492 else 493 return 0; 494} 495 496int dma_fence_get_status(struct dma_fence *fence); 497 498/** 499 * dma_fence_set_error - flag an error condition on the fence 500 * @fence: the dma_fence 501 * @error: the error to store 502 * 503 * Drivers can supply an optional error status condition before they signal 504 * the fence, to indicate that the fence was completed due to an error 505 * rather than success. This must be set before signaling (so that the value 506 * is visible before any waiters on the signal callback are woken). This 507 * helper exists to help catching erroneous setting of #dma_fence.error. 508 */ 509static inline void dma_fence_set_error(struct dma_fence *fence, 510 int error) 511{ 512 WARN_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)); 513 WARN_ON(error >= 0 || error < -MAX_ERRNO); 514 515 fence->error = error; 516} 517 518signed long dma_fence_wait_timeout(struct dma_fence *, 519 bool intr, signed long timeout); 520signed long dma_fence_wait_any_timeout(struct dma_fence **fences, 521 uint32_t count, 522 bool intr, signed long timeout, 523 uint32_t *idx); 524 525/** 526 * dma_fence_wait - sleep until the fence gets signaled 527 * @fence: the fence to wait on 528 * @intr: if true, do an interruptible wait 529 * 530 * This function will return -ERESTARTSYS if interrupted by a signal, 531 * or 0 if the fence was signaled. Other error values may be 532 * returned on custom implementations. 533 * 534 * Performs a synchronous wait on this fence. It is assumed the caller 535 * directly or indirectly holds a reference to the fence, otherwise the 536 * fence might be freed before return, resulting in undefined behavior. 537 * 538 * See also dma_fence_wait_timeout() and dma_fence_wait_any_timeout(). 539 */ 540static inline signed long dma_fence_wait(struct dma_fence *fence, bool intr) 541{ 542 signed long ret; 543 544 /* Since dma_fence_wait_timeout cannot timeout with 545 * MAX_SCHEDULE_TIMEOUT, only valid return values are 546 * -ERESTARTSYS and MAX_SCHEDULE_TIMEOUT. 547 */ 548 ret = dma_fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT); 549 550 return ret < 0 ? ret : 0; 551} 552 553struct dma_fence *dma_fence_get_stub(void); 554u64 dma_fence_context_alloc(unsigned num); 555 556#define DMA_FENCE_TRACE(f, fmt, args...) \ 557 do { \ 558 struct dma_fence *__ff = (f); \ 559 if (IS_ENABLED(CONFIG_DMA_FENCE_TRACE)) \ 560 pr_info("f %llu#%llu: " fmt, \ 561 __ff->context, __ff->seqno, ##args); \ 562 } while (0) 563 564#define DMA_FENCE_WARN(f, fmt, args...) \ 565 do { \ 566 struct dma_fence *__ff = (f); \ 567 pr_warn("f %llu#%llu: " fmt, __ff->context, __ff->seqno,\ 568 ##args); \ 569 } while (0) 570 571#define DMA_FENCE_ERR(f, fmt, args...) \ 572 do { \ 573 struct dma_fence *__ff = (f); \ 574 pr_err("f %llu#%llu: " fmt, __ff->context, __ff->seqno, \ 575 ##args); \ 576 } while (0) 577 578#endif /* __LINUX_DMA_FENCE_H */ 579