1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Wound/Wait Mutexes: blocking mutual exclusion locks with deadlock avoidance 4 * 5 * Original mutex implementation started by Ingo Molnar: 6 * 7 * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 8 * 9 * Wait/Die implementation: 10 * Copyright (C) 2013 Canonical Ltd. 11 * Choice of algorithm: 12 * Copyright (C) 2018 WMWare Inc. 13 * 14 * This file contains the main data structure and API definitions. 15 */ 16 17#ifndef __LINUX_WW_MUTEX_H 18#define __LINUX_WW_MUTEX_H 19 20#include <linux/mutex.h> 21#include <linux/rtmutex.h> 22 23#if defined(CONFIG_DEBUG_MUTEXES) || \ 24 (defined(CONFIG_PREEMPT_RT) && defined(CONFIG_DEBUG_RT_MUTEXES)) 25#define DEBUG_WW_MUTEXES 26#endif 27 28#ifndef CONFIG_PREEMPT_RT 29#define WW_MUTEX_BASE mutex 30#define ww_mutex_base_init(l,n,k) __mutex_init(l,n,k) 31#define ww_mutex_base_trylock(l) mutex_trylock(l) 32#define ww_mutex_base_is_locked(b) mutex_is_locked((b)) 33#else 34#define WW_MUTEX_BASE rt_mutex 35#define ww_mutex_base_init(l,n,k) __rt_mutex_init(l,n,k) 36#define ww_mutex_base_trylock(l) rt_mutex_trylock(l) 37#define ww_mutex_base_is_locked(b) rt_mutex_base_is_locked(&(b)->rtmutex) 38#endif 39 40struct ww_class { 41 atomic_long_t stamp; 42 struct lock_class_key acquire_key; 43 struct lock_class_key mutex_key; 44 const char *acquire_name; 45 const char *mutex_name; 46 unsigned int is_wait_die; 47}; 48 49struct ww_mutex { 50 struct WW_MUTEX_BASE base; 51 struct ww_acquire_ctx *ctx; 52#ifdef DEBUG_WW_MUTEXES 53 struct ww_class *ww_class; 54#endif 55}; 56 57struct ww_acquire_ctx { 58 struct task_struct *task; 59 unsigned long stamp; 60 unsigned int acquired; 61 unsigned short wounded; 62 unsigned short is_wait_die; 63#ifdef DEBUG_WW_MUTEXES 64 unsigned int done_acquire; 65 struct ww_class *ww_class; 66 void *contending_lock; 67#endif 68#ifdef CONFIG_DEBUG_LOCK_ALLOC 69 struct lockdep_map dep_map; 70#endif 71#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH 72 unsigned int deadlock_inject_interval; 73 unsigned int deadlock_inject_countdown; 74#endif 75}; 76 77#define __WW_CLASS_INITIALIZER(ww_class, _is_wait_die) \ 78 { .stamp = ATOMIC_LONG_INIT(0) \ 79 , .acquire_name = #ww_class "_acquire" \ 80 , .mutex_name = #ww_class "_mutex" \ 81 , .is_wait_die = _is_wait_die } 82 83#define DEFINE_WD_CLASS(classname) \ 84 struct ww_class classname = __WW_CLASS_INITIALIZER(classname, 1) 85 86#define DEFINE_WW_CLASS(classname) \ 87 struct ww_class classname = __WW_CLASS_INITIALIZER(classname, 0) 88 89/** 90 * ww_mutex_init - initialize the w/w mutex 91 * @lock: the mutex to be initialized 92 * @ww_class: the w/w class the mutex should belong to 93 * 94 * Initialize the w/w mutex to unlocked state and associate it with the given 95 * class. Static define macro for w/w mutex is not provided and this function 96 * is the only way to properly initialize the w/w mutex. 97 * 98 * It is not allowed to initialize an already locked mutex. 99 */ 100static inline void ww_mutex_init(struct ww_mutex *lock, 101 struct ww_class *ww_class) 102{ 103 ww_mutex_base_init(&lock->base, ww_class->mutex_name, &ww_class->mutex_key); 104 lock->ctx = NULL; 105#ifdef DEBUG_WW_MUTEXES 106 lock->ww_class = ww_class; 107#endif 108} 109 110/** 111 * ww_acquire_init - initialize a w/w acquire context 112 * @ctx: w/w acquire context to initialize 113 * @ww_class: w/w class of the context 114 * 115 * Initializes an context to acquire multiple mutexes of the given w/w class. 116 * 117 * Context-based w/w mutex acquiring can be done in any order whatsoever within 118 * a given lock class. Deadlocks will be detected and handled with the 119 * wait/die logic. 120 * 121 * Mixing of context-based w/w mutex acquiring and single w/w mutex locking can 122 * result in undetected deadlocks and is so forbidden. Mixing different contexts 123 * for the same w/w class when acquiring mutexes can also result in undetected 124 * deadlocks, and is hence also forbidden. Both types of abuse will be caught by 125 * enabling CONFIG_PROVE_LOCKING. 126 * 127 * Nesting of acquire contexts for _different_ w/w classes is possible, subject 128 * to the usual locking rules between different lock classes. 129 * 130 * An acquire context must be released with ww_acquire_fini by the same task 131 * before the memory is freed. It is recommended to allocate the context itself 132 * on the stack. 133 */ 134static inline void ww_acquire_init(struct ww_acquire_ctx *ctx, 135 struct ww_class *ww_class) 136{ 137 ctx->task = current; 138 ctx->stamp = atomic_long_inc_return_relaxed(&ww_class->stamp); 139 ctx->acquired = 0; 140 ctx->wounded = false; 141 ctx->is_wait_die = ww_class->is_wait_die; 142#ifdef DEBUG_WW_MUTEXES 143 ctx->ww_class = ww_class; 144 ctx->done_acquire = 0; 145 ctx->contending_lock = NULL; 146#endif 147#ifdef CONFIG_DEBUG_LOCK_ALLOC 148 debug_check_no_locks_freed((void *)ctx, sizeof(*ctx)); 149 lockdep_init_map(&ctx->dep_map, ww_class->acquire_name, 150 &ww_class->acquire_key, 0); 151 mutex_acquire(&ctx->dep_map, 0, 0, _RET_IP_); 152#endif 153#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH 154 ctx->deadlock_inject_interval = 1; 155 ctx->deadlock_inject_countdown = ctx->stamp & 0xf; 156#endif 157} 158 159/** 160 * ww_acquire_done - marks the end of the acquire phase 161 * @ctx: the acquire context 162 * 163 * Marks the end of the acquire phase, any further w/w mutex lock calls using 164 * this context are forbidden. 165 * 166 * Calling this function is optional, it is just useful to document w/w mutex 167 * code and clearly designated the acquire phase from actually using the locked 168 * data structures. 169 */ 170static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) 171{ 172#ifdef DEBUG_WW_MUTEXES 173 lockdep_assert_held(ctx); 174 175 DEBUG_LOCKS_WARN_ON(ctx->done_acquire); 176 ctx->done_acquire = 1; 177#endif 178} 179 180/** 181 * ww_acquire_fini - releases a w/w acquire context 182 * @ctx: the acquire context to free 183 * 184 * Releases a w/w acquire context. This must be called _after_ all acquired w/w 185 * mutexes have been released with ww_mutex_unlock. 186 */ 187static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx) 188{ 189#ifdef CONFIG_DEBUG_LOCK_ALLOC 190 mutex_release(&ctx->dep_map, _THIS_IP_); 191#endif 192#ifdef DEBUG_WW_MUTEXES 193 DEBUG_LOCKS_WARN_ON(ctx->acquired); 194 if (!IS_ENABLED(CONFIG_PROVE_LOCKING)) 195 /* 196 * lockdep will normally handle this, 197 * but fail without anyway 198 */ 199 ctx->done_acquire = 1; 200 201 if (!IS_ENABLED(CONFIG_DEBUG_LOCK_ALLOC)) 202 /* ensure ww_acquire_fini will still fail if called twice */ 203 ctx->acquired = ~0U; 204#endif 205} 206 207/** 208 * ww_mutex_lock - acquire the w/w mutex 209 * @lock: the mutex to be acquired 210 * @ctx: w/w acquire context, or NULL to acquire only a single lock. 211 * 212 * Lock the w/w mutex exclusively for this task. 213 * 214 * Deadlocks within a given w/w class of locks are detected and handled with the 215 * wait/die algorithm. If the lock isn't immediately available this function 216 * will either sleep until it is (wait case). Or it selects the current context 217 * for backing off by returning -EDEADLK (die case). Trying to acquire the 218 * same lock with the same context twice is also detected and signalled by 219 * returning -EALREADY. Returns 0 if the mutex was successfully acquired. 220 * 221 * In the die case the caller must release all currently held w/w mutexes for 222 * the given context and then wait for this contending lock to be available by 223 * calling ww_mutex_lock_slow. Alternatively callers can opt to not acquire this 224 * lock and proceed with trying to acquire further w/w mutexes (e.g. when 225 * scanning through lru lists trying to free resources). 226 * 227 * The mutex must later on be released by the same task that 228 * acquired it. The task may not exit without first unlocking the mutex. Also, 229 * kernel memory where the mutex resides must not be freed with the mutex still 230 * locked. The mutex must first be initialized (or statically defined) before it 231 * can be locked. memset()-ing the mutex to 0 is not allowed. The mutex must be 232 * of the same w/w lock class as was used to initialize the acquire context. 233 * 234 * A mutex acquired with this function must be released with ww_mutex_unlock. 235 */ 236extern int /* __must_check */ ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx); 237 238/** 239 * ww_mutex_lock_interruptible - acquire the w/w mutex, interruptible 240 * @lock: the mutex to be acquired 241 * @ctx: w/w acquire context 242 * 243 * Lock the w/w mutex exclusively for this task. 244 * 245 * Deadlocks within a given w/w class of locks are detected and handled with the 246 * wait/die algorithm. If the lock isn't immediately available this function 247 * will either sleep until it is (wait case). Or it selects the current context 248 * for backing off by returning -EDEADLK (die case). Trying to acquire the 249 * same lock with the same context twice is also detected and signalled by 250 * returning -EALREADY. Returns 0 if the mutex was successfully acquired. If a 251 * signal arrives while waiting for the lock then this function returns -EINTR. 252 * 253 * In the die case the caller must release all currently held w/w mutexes for 254 * the given context and then wait for this contending lock to be available by 255 * calling ww_mutex_lock_slow_interruptible. Alternatively callers can opt to 256 * not acquire this lock and proceed with trying to acquire further w/w mutexes 257 * (e.g. when scanning through lru lists trying to free resources). 258 * 259 * The mutex must later on be released by the same task that 260 * acquired it. The task may not exit without first unlocking the mutex. Also, 261 * kernel memory where the mutex resides must not be freed with the mutex still 262 * locked. The mutex must first be initialized (or statically defined) before it 263 * can be locked. memset()-ing the mutex to 0 is not allowed. The mutex must be 264 * of the same w/w lock class as was used to initialize the acquire context. 265 * 266 * A mutex acquired with this function must be released with ww_mutex_unlock. 267 */ 268extern int __must_check ww_mutex_lock_interruptible(struct ww_mutex *lock, 269 struct ww_acquire_ctx *ctx); 270 271/** 272 * ww_mutex_lock_slow - slowpath acquiring of the w/w mutex 273 * @lock: the mutex to be acquired 274 * @ctx: w/w acquire context 275 * 276 * Acquires a w/w mutex with the given context after a die case. This function 277 * will sleep until the lock becomes available. 278 * 279 * The caller must have released all w/w mutexes already acquired with the 280 * context and then call this function on the contended lock. 281 * 282 * Afterwards the caller may continue to (re)acquire the other w/w mutexes it 283 * needs with ww_mutex_lock. Note that the -EALREADY return code from 284 * ww_mutex_lock can be used to avoid locking this contended mutex twice. 285 * 286 * It is forbidden to call this function with any other w/w mutexes associated 287 * with the context held. It is forbidden to call this on anything else than the 288 * contending mutex. 289 * 290 * Note that the slowpath lock acquiring can also be done by calling 291 * ww_mutex_lock directly. This function here is simply to help w/w mutex 292 * locking code readability by clearly denoting the slowpath. 293 */ 294static inline void 295ww_mutex_lock_slow(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) 296{ 297 int ret; 298#ifdef DEBUG_WW_MUTEXES 299 DEBUG_LOCKS_WARN_ON(!ctx->contending_lock); 300#endif 301 ret = ww_mutex_lock(lock, ctx); 302 (void)ret; 303} 304 305/** 306 * ww_mutex_lock_slow_interruptible - slowpath acquiring of the w/w mutex, interruptible 307 * @lock: the mutex to be acquired 308 * @ctx: w/w acquire context 309 * 310 * Acquires a w/w mutex with the given context after a die case. This function 311 * will sleep until the lock becomes available and returns 0 when the lock has 312 * been acquired. If a signal arrives while waiting for the lock then this 313 * function returns -EINTR. 314 * 315 * The caller must have released all w/w mutexes already acquired with the 316 * context and then call this function on the contended lock. 317 * 318 * Afterwards the caller may continue to (re)acquire the other w/w mutexes it 319 * needs with ww_mutex_lock. Note that the -EALREADY return code from 320 * ww_mutex_lock can be used to avoid locking this contended mutex twice. 321 * 322 * It is forbidden to call this function with any other w/w mutexes associated 323 * with the given context held. It is forbidden to call this on anything else 324 * than the contending mutex. 325 * 326 * Note that the slowpath lock acquiring can also be done by calling 327 * ww_mutex_lock_interruptible directly. This function here is simply to help 328 * w/w mutex locking code readability by clearly denoting the slowpath. 329 */ 330static inline int __must_check 331ww_mutex_lock_slow_interruptible(struct ww_mutex *lock, 332 struct ww_acquire_ctx *ctx) 333{ 334#ifdef DEBUG_WW_MUTEXES 335 DEBUG_LOCKS_WARN_ON(!ctx->contending_lock); 336#endif 337 return ww_mutex_lock_interruptible(lock, ctx); 338} 339 340extern void ww_mutex_unlock(struct ww_mutex *lock); 341 342/** 343 * ww_mutex_trylock - tries to acquire the w/w mutex without acquire context 344 * @lock: mutex to lock 345 * 346 * Trylocks a mutex without acquire context, so no deadlock detection is 347 * possible. Returns 1 if the mutex has been acquired successfully, 0 otherwise. 348 */ 349static inline int __must_check ww_mutex_trylock(struct ww_mutex *lock) 350{ 351 return ww_mutex_base_trylock(&lock->base); 352} 353 354/*** 355 * ww_mutex_destroy - mark a w/w mutex unusable 356 * @lock: the mutex to be destroyed 357 * 358 * This function marks the mutex uninitialized, and any subsequent 359 * use of the mutex is forbidden. The mutex must not be locked when 360 * this function is called. 361 */ 362static inline void ww_mutex_destroy(struct ww_mutex *lock) 363{ 364#ifndef CONFIG_PREEMPT_RT 365 mutex_destroy(&lock->base); 366#endif 367} 368 369/** 370 * ww_mutex_is_locked - is the w/w mutex locked 371 * @lock: the mutex to be queried 372 * 373 * Returns 1 if the mutex is locked, 0 if unlocked. 374 */ 375static inline bool ww_mutex_is_locked(struct ww_mutex *lock) 376{ 377 return ww_mutex_base_is_locked(&lock->base); 378} 379 380#endif 381