linux/kernel/rtmutex.c
<<
>>
Prefs
   1/*
   2 * RT-Mutexes: simple blocking mutual exclusion locks with PI support
   3 *
   4 * started by Ingo Molnar and Thomas Gleixner.
   5 *
   6 *  Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
   7 *  Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
   8 *  Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
   9 *  Copyright (C) 2006 Esben Nielsen
  10 *
  11 *  See Documentation/rt-mutex-design.txt for details.
  12 */
  13#include <linux/spinlock.h>
  14#include <linux/module.h>
  15#include <linux/sched.h>
  16#include <linux/timer.h>
  17
  18#include "rtmutex_common.h"
  19
  20/*
  21 * lock->owner state tracking:
  22 *
  23 * lock->owner holds the task_struct pointer of the owner. Bit 0
  24 * is used to keep track of the "lock has waiters" state.
  25 *
  26 * owner        bit0
  27 * NULL         0       lock is free (fast acquire possible)
  28 * NULL         1       lock is free and has waiters and the top waiter
  29 *                              is going to take the lock*
  30 * taskpointer  0       lock is held (fast release possible)
  31 * taskpointer  1       lock is held and has waiters**
  32 *
  33 * The fast atomic compare exchange based acquire and release is only
  34 * possible when bit 0 of lock->owner is 0.
  35 *
  36 * (*) It also can be a transitional state when grabbing the lock
  37 * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
  38 * we need to set the bit0 before looking at the lock, and the owner may be
  39 * NULL in this small time, hence this can be a transitional state.
  40 *
  41 * (**) There is a small time when bit 0 is set but there are no
  42 * waiters. This can happen when grabbing the lock in the slow path.
  43 * To prevent a cmpxchg of the owner releasing the lock, we need to
  44 * set this bit before looking at the lock.
  45 */
  46
  47static void
  48rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
  49{
  50        unsigned long val = (unsigned long)owner;
  51
  52        if (rt_mutex_has_waiters(lock))
  53                val |= RT_MUTEX_HAS_WAITERS;
  54
  55        lock->owner = (struct task_struct *)val;
  56}
  57
  58static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
  59{
  60        lock->owner = (struct task_struct *)
  61                        ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
  62}
  63
  64static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
  65{
  66        if (!rt_mutex_has_waiters(lock))
  67                clear_rt_mutex_waiters(lock);
  68}
  69
  70/*
  71 * We can speed up the acquire/release, if the architecture
  72 * supports cmpxchg and if there's no debugging state to be set up
  73 */
  74#if defined(__HAVE_ARCH_CMPXCHG) && !defined(CONFIG_DEBUG_RT_MUTEXES)
  75# define rt_mutex_cmpxchg(l,c,n)        (cmpxchg(&l->owner, c, n) == c)
  76static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
  77{
  78        unsigned long owner, *p = (unsigned long *) &lock->owner;
  79
  80        do {
  81                owner = *p;
  82        } while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner);
  83}
  84#else
  85# define rt_mutex_cmpxchg(l,c,n)        (0)
  86static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
  87{
  88        lock->owner = (struct task_struct *)
  89                        ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
  90}
  91#endif
  92
  93/*
  94 * Calculate task priority from the waiter list priority
  95 *
  96 * Return task->normal_prio when the waiter list is empty or when
  97 * the waiter is not allowed to do priority boosting
  98 */
  99int rt_mutex_getprio(struct task_struct *task)
 100{
 101        if (likely(!task_has_pi_waiters(task)))
 102                return task->normal_prio;
 103
 104        return min(task_top_pi_waiter(task)->pi_list_entry.prio,
 105                   task->normal_prio);
 106}
 107
 108/*
 109 * Adjust the priority of a task, after its pi_waiters got modified.
 110 *
 111 * This can be both boosting and unboosting. task->pi_lock must be held.
 112 */
 113static void __rt_mutex_adjust_prio(struct task_struct *task)
 114{
 115        int prio = rt_mutex_getprio(task);
 116
 117        if (task->prio != prio)
 118                rt_mutex_setprio(task, prio);
 119}
 120
 121/*
 122 * Adjust task priority (undo boosting). Called from the exit path of
 123 * rt_mutex_slowunlock() and rt_mutex_slowlock().
 124 *
 125 * (Note: We do this outside of the protection of lock->wait_lock to
 126 * allow the lock to be taken while or before we readjust the priority
 127 * of task. We do not use the spin_xx_mutex() variants here as we are
 128 * outside of the debug path.)
 129 */
 130static void rt_mutex_adjust_prio(struct task_struct *task)
 131{
 132        unsigned long flags;
 133
 134        raw_spin_lock_irqsave(&task->pi_lock, flags);
 135        __rt_mutex_adjust_prio(task);
 136        raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 137}
 138
 139/*
 140 * Max number of times we'll walk the boosting chain:
 141 */
 142int max_lock_depth = 1024;
 143
 144/*
 145 * Adjust the priority chain. Also used for deadlock detection.
 146 * Decreases task's usage by one - may thus free the task.
 147 * Returns 0 or -EDEADLK.
 148 */
 149static int rt_mutex_adjust_prio_chain(struct task_struct *task,
 150                                      int deadlock_detect,
 151                                      struct rt_mutex *orig_lock,
 152                                      struct rt_mutex_waiter *orig_waiter,
 153                                      struct task_struct *top_task)
 154{
 155        struct rt_mutex *lock;
 156        struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
 157        int detect_deadlock, ret = 0, depth = 0;
 158        unsigned long flags;
 159
 160        detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter,
 161                                                         deadlock_detect);
 162
 163        /*
 164         * The (de)boosting is a step by step approach with a lot of
 165         * pitfalls. We want this to be preemptible and we want hold a
 166         * maximum of two locks per step. So we have to check
 167         * carefully whether things change under us.
 168         */
 169 again:
 170        if (++depth > max_lock_depth) {
 171                static int prev_max;
 172
 173                /*
 174                 * Print this only once. If the admin changes the limit,
 175                 * print a new message when reaching the limit again.
 176                 */
 177                if (prev_max != max_lock_depth) {
 178                        prev_max = max_lock_depth;
 179                        printk(KERN_WARNING "Maximum lock depth %d reached "
 180                               "task: %s (%d)\n", max_lock_depth,
 181                               top_task->comm, task_pid_nr(top_task));
 182                }
 183                put_task_struct(task);
 184
 185                return deadlock_detect ? -EDEADLK : 0;
 186        }
 187 retry:
 188        /*
 189         * Task can not go away as we did a get_task() before !
 190         */
 191        raw_spin_lock_irqsave(&task->pi_lock, flags);
 192
 193        waiter = task->pi_blocked_on;
 194        /*
 195         * Check whether the end of the boosting chain has been
 196         * reached or the state of the chain has changed while we
 197         * dropped the locks.
 198         */
 199        if (!waiter)
 200                goto out_unlock_pi;
 201
 202        /*
 203         * Check the orig_waiter state. After we dropped the locks,
 204         * the previous owner of the lock might have released the lock.
 205         */
 206        if (orig_waiter && !rt_mutex_owner(orig_lock))
 207                goto out_unlock_pi;
 208
 209        /*
 210         * Drop out, when the task has no waiters. Note,
 211         * top_waiter can be NULL, when we are in the deboosting
 212         * mode!
 213         */
 214        if (top_waiter && (!task_has_pi_waiters(task) ||
 215                           top_waiter != task_top_pi_waiter(task)))
 216                goto out_unlock_pi;
 217
 218        /*
 219         * When deadlock detection is off then we check, if further
 220         * priority adjustment is necessary.
 221         */
 222        if (!detect_deadlock && waiter->list_entry.prio == task->prio)
 223                goto out_unlock_pi;
 224
 225        lock = waiter->lock;
 226        if (!raw_spin_trylock(&lock->wait_lock)) {
 227                raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 228                cpu_relax();
 229                goto retry;
 230        }
 231
 232        /* Deadlock detection */
 233        if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
 234                debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);
 235                raw_spin_unlock(&lock->wait_lock);
 236                ret = deadlock_detect ? -EDEADLK : 0;
 237                goto out_unlock_pi;
 238        }
 239
 240        top_waiter = rt_mutex_top_waiter(lock);
 241
 242        /* Requeue the waiter */
 243        plist_del(&waiter->list_entry, &lock->wait_list);
 244        waiter->list_entry.prio = task->prio;
 245        plist_add(&waiter->list_entry, &lock->wait_list);
 246
 247        /* Release the task */
 248        raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 249        if (!rt_mutex_owner(lock)) {
 250                /*
 251                 * If the requeue above changed the top waiter, then we need
 252                 * to wake the new top waiter up to try to get the lock.
 253                 */
 254
 255                if (top_waiter != rt_mutex_top_waiter(lock))
 256                        wake_up_process(rt_mutex_top_waiter(lock)->task);
 257                raw_spin_unlock(&lock->wait_lock);
 258                goto out_put_task;
 259        }
 260        put_task_struct(task);
 261
 262        /* Grab the next task */
 263        task = rt_mutex_owner(lock);
 264        get_task_struct(task);
 265        raw_spin_lock_irqsave(&task->pi_lock, flags);
 266
 267        if (waiter == rt_mutex_top_waiter(lock)) {
 268                /* Boost the owner */
 269                plist_del(&top_waiter->pi_list_entry, &task->pi_waiters);
 270                waiter->pi_list_entry.prio = waiter->list_entry.prio;
 271                plist_add(&waiter->pi_list_entry, &task->pi_waiters);
 272                __rt_mutex_adjust_prio(task);
 273
 274        } else if (top_waiter == waiter) {
 275                /* Deboost the owner */
 276                plist_del(&waiter->pi_list_entry, &task->pi_waiters);
 277                waiter = rt_mutex_top_waiter(lock);
 278                waiter->pi_list_entry.prio = waiter->list_entry.prio;
 279                plist_add(&waiter->pi_list_entry, &task->pi_waiters);
 280                __rt_mutex_adjust_prio(task);
 281        }
 282
 283        raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 284
 285        top_waiter = rt_mutex_top_waiter(lock);
 286        raw_spin_unlock(&lock->wait_lock);
 287
 288        if (!detect_deadlock && waiter != top_waiter)
 289                goto out_put_task;
 290
 291        goto again;
 292
 293 out_unlock_pi:
 294        raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 295 out_put_task:
 296        put_task_struct(task);
 297
 298        return ret;
 299}
 300
 301/*
 302 * Try to take an rt-mutex
 303 *
 304 * Must be called with lock->wait_lock held.
 305 *
 306 * @lock:   the lock to be acquired.
 307 * @task:   the task which wants to acquire the lock
 308 * @waiter: the waiter that is queued to the lock's wait list. (could be NULL)
 309 */
 310static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
 311                struct rt_mutex_waiter *waiter)
 312{
 313        /*
 314         * We have to be careful here if the atomic speedups are
 315         * enabled, such that, when
 316         *  - no other waiter is on the lock
 317         *  - the lock has been released since we did the cmpxchg
 318         * the lock can be released or taken while we are doing the
 319         * checks and marking the lock with RT_MUTEX_HAS_WAITERS.
 320         *
 321         * The atomic acquire/release aware variant of
 322         * mark_rt_mutex_waiters uses a cmpxchg loop. After setting
 323         * the WAITERS bit, the atomic release / acquire can not
 324         * happen anymore and lock->wait_lock protects us from the
 325         * non-atomic case.
 326         *
 327         * Note, that this might set lock->owner =
 328         * RT_MUTEX_HAS_WAITERS in the case the lock is not contended
 329         * any more. This is fixed up when we take the ownership.
 330         * This is the transitional state explained at the top of this file.
 331         */
 332        mark_rt_mutex_waiters(lock);
 333
 334        if (rt_mutex_owner(lock))
 335                return 0;
 336
 337        /*
 338         * It will get the lock because of one of these conditions:
 339         * 1) there is no waiter
 340         * 2) higher priority than waiters
 341         * 3) it is top waiter
 342         */
 343        if (rt_mutex_has_waiters(lock)) {
 344                if (task->prio >= rt_mutex_top_waiter(lock)->list_entry.prio) {
 345                        if (!waiter || waiter != rt_mutex_top_waiter(lock))
 346                                return 0;
 347                }
 348        }
 349
 350        if (waiter || rt_mutex_has_waiters(lock)) {
 351                unsigned long flags;
 352                struct rt_mutex_waiter *top;
 353
 354                raw_spin_lock_irqsave(&task->pi_lock, flags);
 355
 356                /* remove the queued waiter. */
 357                if (waiter) {
 358                        plist_del(&waiter->list_entry, &lock->wait_list);
 359                        task->pi_blocked_on = NULL;
 360                }
 361
 362                /*
 363                 * We have to enqueue the top waiter(if it exists) into
 364                 * task->pi_waiters list.
 365                 */
 366                if (rt_mutex_has_waiters(lock)) {
 367                        top = rt_mutex_top_waiter(lock);
 368                        top->pi_list_entry.prio = top->list_entry.prio;
 369                        plist_add(&top->pi_list_entry, &task->pi_waiters);
 370                }
 371                raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 372        }
 373
 374        /* We got the lock. */
 375        debug_rt_mutex_lock(lock);
 376
 377        rt_mutex_set_owner(lock, task);
 378
 379        rt_mutex_deadlock_account_lock(lock, task);
 380
 381        return 1;
 382}
 383
 384/*
 385 * Task blocks on lock.
 386 *
 387 * Prepare waiter and propagate pi chain
 388 *
 389 * This must be called with lock->wait_lock held.
 390 */
 391static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
 392                                   struct rt_mutex_waiter *waiter,
 393                                   struct task_struct *task,
 394                                   int detect_deadlock)
 395{
 396        struct task_struct *owner = rt_mutex_owner(lock);
 397        struct rt_mutex_waiter *top_waiter = waiter;
 398        unsigned long flags;
 399        int chain_walk = 0, res;
 400
 401        raw_spin_lock_irqsave(&task->pi_lock, flags);
 402        __rt_mutex_adjust_prio(task);
 403        waiter->task = task;
 404        waiter->lock = lock;
 405        plist_node_init(&waiter->list_entry, task->prio);
 406        plist_node_init(&waiter->pi_list_entry, task->prio);
 407
 408        /* Get the top priority waiter on the lock */
 409        if (rt_mutex_has_waiters(lock))
 410                top_waiter = rt_mutex_top_waiter(lock);
 411        plist_add(&waiter->list_entry, &lock->wait_list);
 412
 413        task->pi_blocked_on = waiter;
 414
 415        raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 416
 417        if (!owner)
 418                return 0;
 419
 420        if (waiter == rt_mutex_top_waiter(lock)) {
 421                raw_spin_lock_irqsave(&owner->pi_lock, flags);
 422                plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
 423                plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
 424
 425                __rt_mutex_adjust_prio(owner);
 426                if (owner->pi_blocked_on)
 427                        chain_walk = 1;
 428                raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
 429        }
 430        else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
 431                chain_walk = 1;
 432
 433        if (!chain_walk)
 434                return 0;
 435
 436        /*
 437         * The owner can't disappear while holding a lock,
 438         * so the owner struct is protected by wait_lock.
 439         * Gets dropped in rt_mutex_adjust_prio_chain()!
 440         */
 441        get_task_struct(owner);
 442
 443        raw_spin_unlock(&lock->wait_lock);
 444
 445        res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
 446                                         task);
 447
 448        raw_spin_lock(&lock->wait_lock);
 449
 450        return res;
 451}
 452
 453/*
 454 * Wake up the next waiter on the lock.
 455 *
 456 * Remove the top waiter from the current tasks waiter list and wake it up.
 457 *
 458 * Called with lock->wait_lock held.
 459 */
 460static void wakeup_next_waiter(struct rt_mutex *lock)
 461{
 462        struct rt_mutex_waiter *waiter;
 463        unsigned long flags;
 464
 465        raw_spin_lock_irqsave(&current->pi_lock, flags);
 466
 467        waiter = rt_mutex_top_waiter(lock);
 468
 469        /*
 470         * Remove it from current->pi_waiters. We do not adjust a
 471         * possible priority boost right now. We execute wakeup in the
 472         * boosted mode and go back to normal after releasing
 473         * lock->wait_lock.
 474         */
 475        plist_del(&waiter->pi_list_entry, &current->pi_waiters);
 476
 477        rt_mutex_set_owner(lock, NULL);
 478
 479        raw_spin_unlock_irqrestore(&current->pi_lock, flags);
 480
 481        wake_up_process(waiter->task);
 482}
 483
 484/*
 485 * Remove a waiter from a lock and give up
 486 *
 487 * Must be called with lock->wait_lock held and
 488 * have just failed to try_to_take_rt_mutex().
 489 */
 490static void remove_waiter(struct rt_mutex *lock,
 491                          struct rt_mutex_waiter *waiter)
 492{
 493        int first = (waiter == rt_mutex_top_waiter(lock));
 494        struct task_struct *owner = rt_mutex_owner(lock);
 495        unsigned long flags;
 496        int chain_walk = 0;
 497
 498        raw_spin_lock_irqsave(&current->pi_lock, flags);
 499        plist_del(&waiter->list_entry, &lock->wait_list);
 500        current->pi_blocked_on = NULL;
 501        raw_spin_unlock_irqrestore(&current->pi_lock, flags);
 502
 503        if (!owner)
 504                return;
 505
 506        if (first) {
 507
 508                raw_spin_lock_irqsave(&owner->pi_lock, flags);
 509
 510                plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
 511
 512                if (rt_mutex_has_waiters(lock)) {
 513                        struct rt_mutex_waiter *next;
 514
 515                        next = rt_mutex_top_waiter(lock);
 516                        plist_add(&next->pi_list_entry, &owner->pi_waiters);
 517                }
 518                __rt_mutex_adjust_prio(owner);
 519
 520                if (owner->pi_blocked_on)
 521                        chain_walk = 1;
 522
 523                raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
 524        }
 525
 526        WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
 527
 528        if (!chain_walk)
 529                return;
 530
 531        /* gets dropped in rt_mutex_adjust_prio_chain()! */
 532        get_task_struct(owner);
 533
 534        raw_spin_unlock(&lock->wait_lock);
 535
 536        rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
 537
 538        raw_spin_lock(&lock->wait_lock);
 539}
 540
 541/*
 542 * Recheck the pi chain, in case we got a priority setting
 543 *
 544 * Called from sched_setscheduler
 545 */
 546void rt_mutex_adjust_pi(struct task_struct *task)
 547{
 548        struct rt_mutex_waiter *waiter;
 549        unsigned long flags;
 550
 551        raw_spin_lock_irqsave(&task->pi_lock, flags);
 552
 553        waiter = task->pi_blocked_on;
 554        if (!waiter || waiter->list_entry.prio == task->prio) {
 555                raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 556                return;
 557        }
 558
 559        raw_spin_unlock_irqrestore(&task->pi_lock, flags);
 560
 561        /* gets dropped in rt_mutex_adjust_prio_chain()! */
 562        get_task_struct(task);
 563        rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task);
 564}
 565
 566/**
 567 * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
 568 * @lock:                the rt_mutex to take
 569 * @state:               the state the task should block in (TASK_INTERRUPTIBLE
 570 *                       or TASK_UNINTERRUPTIBLE)
 571 * @timeout:             the pre-initialized and started timer, or NULL for none
 572 * @waiter:              the pre-initialized rt_mutex_waiter
 573 *
 574 * lock->wait_lock must be held by the caller.
 575 */
 576static int __sched
 577__rt_mutex_slowlock(struct rt_mutex *lock, int state,
 578                    struct hrtimer_sleeper *timeout,
 579                    struct rt_mutex_waiter *waiter)
 580{
 581        int ret = 0;
 582
 583        for (;;) {
 584                /* Try to acquire the lock: */
 585                if (try_to_take_rt_mutex(lock, current, waiter))
 586                        break;
 587
 588                /*
 589                 * TASK_INTERRUPTIBLE checks for signals and
 590                 * timeout. Ignored otherwise.
 591                 */
 592                if (unlikely(state == TASK_INTERRUPTIBLE)) {
 593                        /* Signal pending? */
 594                        if (signal_pending(current))
 595                                ret = -EINTR;
 596                        if (timeout && !timeout->task)
 597                                ret = -ETIMEDOUT;
 598                        if (ret)
 599                                break;
 600                }
 601
 602                raw_spin_unlock(&lock->wait_lock);
 603
 604                debug_rt_mutex_print_deadlock(waiter);
 605
 606                schedule_rt_mutex(lock);
 607
 608                raw_spin_lock(&lock->wait_lock);
 609                set_current_state(state);
 610        }
 611
 612        return ret;
 613}
 614
 615/*
 616 * Slow path lock function:
 617 */
 618static int __sched
 619rt_mutex_slowlock(struct rt_mutex *lock, int state,
 620                  struct hrtimer_sleeper *timeout,
 621                  int detect_deadlock)
 622{
 623        struct rt_mutex_waiter waiter;
 624        int ret = 0;
 625
 626        debug_rt_mutex_init_waiter(&waiter);
 627
 628        raw_spin_lock(&lock->wait_lock);
 629
 630        /* Try to acquire the lock again: */
 631        if (try_to_take_rt_mutex(lock, current, NULL)) {
 632                raw_spin_unlock(&lock->wait_lock);
 633                return 0;
 634        }
 635
 636        set_current_state(state);
 637
 638        /* Setup the timer, when timeout != NULL */
 639        if (unlikely(timeout)) {
 640                hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
 641                if (!hrtimer_active(&timeout->timer))
 642                        timeout->task = NULL;
 643        }
 644
 645        ret = task_blocks_on_rt_mutex(lock, &waiter, current, detect_deadlock);
 646
 647        if (likely(!ret))
 648                ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
 649
 650        set_current_state(TASK_RUNNING);
 651
 652        if (unlikely(ret))
 653                remove_waiter(lock, &waiter);
 654
 655        /*
 656         * try_to_take_rt_mutex() sets the waiter bit
 657         * unconditionally. We might have to fix that up.
 658         */
 659        fixup_rt_mutex_waiters(lock);
 660
 661        raw_spin_unlock(&lock->wait_lock);
 662
 663        /* Remove pending timer: */
 664        if (unlikely(timeout))
 665                hrtimer_cancel(&timeout->timer);
 666
 667        debug_rt_mutex_free_waiter(&waiter);
 668
 669        return ret;
 670}
 671
 672/*
 673 * Slow path try-lock function:
 674 */
 675static inline int
 676rt_mutex_slowtrylock(struct rt_mutex *lock)
 677{
 678        int ret = 0;
 679
 680        raw_spin_lock(&lock->wait_lock);
 681
 682        if (likely(rt_mutex_owner(lock) != current)) {
 683
 684                ret = try_to_take_rt_mutex(lock, current, NULL);
 685                /*
 686                 * try_to_take_rt_mutex() sets the lock waiters
 687                 * bit unconditionally. Clean this up.
 688                 */
 689                fixup_rt_mutex_waiters(lock);
 690        }
 691
 692        raw_spin_unlock(&lock->wait_lock);
 693
 694        return ret;
 695}
 696
 697/*
 698 * Slow path to release a rt-mutex:
 699 */
 700static void __sched
 701rt_mutex_slowunlock(struct rt_mutex *lock)
 702{
 703        raw_spin_lock(&lock->wait_lock);
 704
 705        debug_rt_mutex_unlock(lock);
 706
 707        rt_mutex_deadlock_account_unlock(current);
 708
 709        if (!rt_mutex_has_waiters(lock)) {
 710                lock->owner = NULL;
 711                raw_spin_unlock(&lock->wait_lock);
 712                return;
 713        }
 714
 715        wakeup_next_waiter(lock);
 716
 717        raw_spin_unlock(&lock->wait_lock);
 718
 719        /* Undo pi boosting if necessary: */
 720        rt_mutex_adjust_prio(current);
 721}
 722
 723/*
 724 * debug aware fast / slowpath lock,trylock,unlock
 725 *
 726 * The atomic acquire/release ops are compiled away, when either the
 727 * architecture does not support cmpxchg or when debugging is enabled.
 728 */
 729static inline int
 730rt_mutex_fastlock(struct rt_mutex *lock, int state,
 731                  int detect_deadlock,
 732                  int (*slowfn)(struct rt_mutex *lock, int state,
 733                                struct hrtimer_sleeper *timeout,
 734                                int detect_deadlock))
 735{
 736        if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
 737                rt_mutex_deadlock_account_lock(lock, current);
 738                return 0;
 739        } else
 740                return slowfn(lock, state, NULL, detect_deadlock);
 741}
 742
 743static inline int
 744rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
 745                        struct hrtimer_sleeper *timeout, int detect_deadlock,
 746                        int (*slowfn)(struct rt_mutex *lock, int state,
 747                                      struct hrtimer_sleeper *timeout,
 748                                      int detect_deadlock))
 749{
 750        if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
 751                rt_mutex_deadlock_account_lock(lock, current);
 752                return 0;
 753        } else
 754                return slowfn(lock, state, timeout, detect_deadlock);
 755}
 756
 757static inline int
 758rt_mutex_fasttrylock(struct rt_mutex *lock,
 759                     int (*slowfn)(struct rt_mutex *lock))
 760{
 761        if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {
 762                rt_mutex_deadlock_account_lock(lock, current);
 763                return 1;
 764        }
 765        return slowfn(lock);
 766}
 767
 768static inline void
 769rt_mutex_fastunlock(struct rt_mutex *lock,
 770                    void (*slowfn)(struct rt_mutex *lock))
 771{
 772        if (likely(rt_mutex_cmpxchg(lock, current, NULL)))
 773                rt_mutex_deadlock_account_unlock(current);
 774        else
 775                slowfn(lock);
 776}
 777
 778/**
 779 * rt_mutex_lock - lock a rt_mutex
 780 *
 781 * @lock: the rt_mutex to be locked
 782 */
 783void __sched rt_mutex_lock(struct rt_mutex *lock)
 784{
 785        might_sleep();
 786
 787        rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock);
 788}
 789EXPORT_SYMBOL_GPL(rt_mutex_lock);
 790
 791/**
 792 * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
 793 *
 794 * @lock:               the rt_mutex to be locked
 795 * @detect_deadlock:    deadlock detection on/off
 796 *
 797 * Returns:
 798 *  0           on success
 799 * -EINTR       when interrupted by a signal
 800 * -EDEADLK     when the lock would deadlock (when deadlock detection is on)
 801 */
 802int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock,
 803                                                 int detect_deadlock)
 804{
 805        might_sleep();
 806
 807        return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE,
 808                                 detect_deadlock, rt_mutex_slowlock);
 809}
 810EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
 811
 812/**
 813 * rt_mutex_timed_lock - lock a rt_mutex interruptible
 814 *                      the timeout structure is provided
 815 *                      by the caller
 816 *
 817 * @lock:               the rt_mutex to be locked
 818 * @timeout:            timeout structure or NULL (no timeout)
 819 * @detect_deadlock:    deadlock detection on/off
 820 *
 821 * Returns:
 822 *  0           on success
 823 * -EINTR       when interrupted by a signal
 824 * -ETIMEDOUT   when the timeout expired
 825 * -EDEADLK     when the lock would deadlock (when deadlock detection is on)
 826 */
 827int
 828rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout,
 829                    int detect_deadlock)
 830{
 831        might_sleep();
 832
 833        return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
 834                                       detect_deadlock, rt_mutex_slowlock);
 835}
 836EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
 837
 838/**
 839 * rt_mutex_trylock - try to lock a rt_mutex
 840 *
 841 * @lock:       the rt_mutex to be locked
 842 *
 843 * Returns 1 on success and 0 on contention
 844 */
 845int __sched rt_mutex_trylock(struct rt_mutex *lock)
 846{
 847        return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
 848}
 849EXPORT_SYMBOL_GPL(rt_mutex_trylock);
 850
 851/**
 852 * rt_mutex_unlock - unlock a rt_mutex
 853 *
 854 * @lock: the rt_mutex to be unlocked
 855 */
 856void __sched rt_mutex_unlock(struct rt_mutex *lock)
 857{
 858        rt_mutex_fastunlock(lock, rt_mutex_slowunlock);
 859}
 860EXPORT_SYMBOL_GPL(rt_mutex_unlock);
 861
 862/**
 863 * rt_mutex_destroy - mark a mutex unusable
 864 * @lock: the mutex to be destroyed
 865 *
 866 * This function marks the mutex uninitialized, and any subsequent
 867 * use of the mutex is forbidden. The mutex must not be locked when
 868 * this function is called.
 869 */
 870void rt_mutex_destroy(struct rt_mutex *lock)
 871{
 872        WARN_ON(rt_mutex_is_locked(lock));
 873#ifdef CONFIG_DEBUG_RT_MUTEXES
 874        lock->magic = NULL;
 875#endif
 876}
 877
 878EXPORT_SYMBOL_GPL(rt_mutex_destroy);
 879
 880/**
 881 * __rt_mutex_init - initialize the rt lock
 882 *
 883 * @lock: the rt lock to be initialized
 884 *
 885 * Initialize the rt lock to unlocked state.
 886 *
 887 * Initializing of a locked rt lock is not allowed
 888 */
 889void __rt_mutex_init(struct rt_mutex *lock, const char *name)
 890{
 891        lock->owner = NULL;
 892        raw_spin_lock_init(&lock->wait_lock);
 893        plist_head_init_raw(&lock->wait_list, &lock->wait_lock);
 894
 895        debug_rt_mutex_init(lock, name);
 896}
 897EXPORT_SYMBOL_GPL(__rt_mutex_init);
 898
 899/**
 900 * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
 901 *                              proxy owner
 902 *
 903 * @lock:       the rt_mutex to be locked
 904 * @proxy_owner:the task to set as owner
 905 *
 906 * No locking. Caller has to do serializing itself
 907 * Special API call for PI-futex support
 908 */
 909void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
 910                                struct task_struct *proxy_owner)
 911{
 912        __rt_mutex_init(lock, NULL);
 913        debug_rt_mutex_proxy_lock(lock, proxy_owner);
 914        rt_mutex_set_owner(lock, proxy_owner);
 915        rt_mutex_deadlock_account_lock(lock, proxy_owner);
 916}
 917
 918/**
 919 * rt_mutex_proxy_unlock - release a lock on behalf of owner
 920 *
 921 * @lock:       the rt_mutex to be locked
 922 *
 923 * No locking. Caller has to do serializing itself
 924 * Special API call for PI-futex support
 925 */
 926void rt_mutex_proxy_unlock(struct rt_mutex *lock,
 927                           struct task_struct *proxy_owner)
 928{
 929        debug_rt_mutex_proxy_unlock(lock);
 930        rt_mutex_set_owner(lock, NULL);
 931        rt_mutex_deadlock_account_unlock(proxy_owner);
 932}
 933
 934/**
 935 * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
 936 * @lock:               the rt_mutex to take
 937 * @waiter:             the pre-initialized rt_mutex_waiter
 938 * @task:               the task to prepare
 939 * @detect_deadlock:    perform deadlock detection (1) or not (0)
 940 *
 941 * Returns:
 942 *  0 - task blocked on lock
 943 *  1 - acquired the lock for task, caller should wake it up
 944 * <0 - error
 945 *
 946 * Special API call for FUTEX_REQUEUE_PI support.
 947 */
 948int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
 949                              struct rt_mutex_waiter *waiter,
 950                              struct task_struct *task, int detect_deadlock)
 951{
 952        int ret;
 953
 954        raw_spin_lock(&lock->wait_lock);
 955
 956        if (try_to_take_rt_mutex(lock, task, NULL)) {
 957                raw_spin_unlock(&lock->wait_lock);
 958                return 1;
 959        }
 960
 961        ret = task_blocks_on_rt_mutex(lock, waiter, task, detect_deadlock);
 962
 963        if (ret && !rt_mutex_owner(lock)) {
 964                /*
 965                 * Reset the return value. We might have
 966                 * returned with -EDEADLK and the owner
 967                 * released the lock while we were walking the
 968                 * pi chain.  Let the waiter sort it out.
 969                 */
 970                ret = 0;
 971        }
 972
 973        if (unlikely(ret))
 974                remove_waiter(lock, waiter);
 975
 976        raw_spin_unlock(&lock->wait_lock);
 977
 978        debug_rt_mutex_print_deadlock(waiter);
 979
 980        return ret;
 981}
 982
 983/**
 984 * rt_mutex_next_owner - return the next owner of the lock
 985 *
 986 * @lock: the rt lock query
 987 *
 988 * Returns the next owner of the lock or NULL
 989 *
 990 * Caller has to serialize against other accessors to the lock
 991 * itself.
 992 *
 993 * Special API call for PI-futex support
 994 */
 995struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
 996{
 997        if (!rt_mutex_has_waiters(lock))
 998                return NULL;
 999
1000        return rt_mutex_top_waiter(lock)->task;
1001}
1002
1003/**
1004 * rt_mutex_finish_proxy_lock() - Complete lock acquisition
1005 * @lock:               the rt_mutex we were woken on
1006 * @to:                 the timeout, null if none. hrtimer should already have
1007 *                      been started.
1008 * @waiter:             the pre-initialized rt_mutex_waiter
1009 * @detect_deadlock:    perform deadlock detection (1) or not (0)
1010 *
1011 * Complete the lock acquisition started our behalf by another thread.
1012 *
1013 * Returns:
1014 *  0 - success
1015 * <0 - error, one of -EINTR, -ETIMEDOUT, or -EDEADLK
1016 *
1017 * Special API call for PI-futex requeue support
1018 */
1019int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
1020                               struct hrtimer_sleeper *to,
1021                               struct rt_mutex_waiter *waiter,
1022                               int detect_deadlock)
1023{
1024        int ret;
1025
1026        raw_spin_lock(&lock->wait_lock);
1027
1028        set_current_state(TASK_INTERRUPTIBLE);
1029
1030        ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
1031
1032        set_current_state(TASK_RUNNING);
1033
1034        if (unlikely(ret))
1035                remove_waiter(lock, waiter);
1036
1037        /*
1038         * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
1039         * have to fix that up.
1040         */
1041        fixup_rt_mutex_waiters(lock);
1042
1043        raw_spin_unlock(&lock->wait_lock);
1044
1045        return ret;
1046}
1047