2 * RT-Mutexes: simple blocking mutual exclusion locks with PI support
4 * started by Ingo Molnar and Thomas Gleixner.
6 * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
7 * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
8 * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
9 * Copyright (C) 2006 Esben Nielsen
11 * See Documentation/locking/rt-mutex-design.txt for details.
13 #include <linux/spinlock.h>
14 #include <linux/export.h>
15 #include <linux/sched/signal.h>
16 #include <linux/sched/rt.h>
17 #include <linux/sched/deadline.h>
18 #include <linux/sched/wake_q.h>
19 #include <linux/sched/debug.h>
20 #include <linux/timer.h>
22 #include "rtmutex_common.h"
25 * lock->owner state tracking:
27 * lock->owner holds the task_struct pointer of the owner. Bit 0
28 * is used to keep track of the "lock has waiters" state.
31 * NULL 0 lock is free (fast acquire possible)
32 * NULL 1 lock is free and has waiters and the top waiter
33 * is going to take the lock*
34 * taskpointer 0 lock is held (fast release possible)
35 * taskpointer 1 lock is held and has waiters**
37 * The fast atomic compare exchange based acquire and release is only
38 * possible when bit 0 of lock->owner is 0.
40 * (*) It also can be a transitional state when grabbing the lock
41 * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
42 * we need to set the bit0 before looking at the lock, and the owner may be
43 * NULL in this small time, hence this can be a transitional state.
45 * (**) There is a small time when bit 0 is set but there are no
46 * waiters. This can happen when grabbing the lock in the slow path.
47 * To prevent a cmpxchg of the owner releasing the lock, we need to
48 * set this bit before looking at the lock.
52 rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
54 unsigned long val = (unsigned long)owner;
56 if (rt_mutex_has_waiters(lock))
57 val |= RT_MUTEX_HAS_WAITERS;
59 lock->owner = (struct task_struct *)val;
62 static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
64 lock->owner = (struct task_struct *)
65 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
68 static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
70 unsigned long owner, *p = (unsigned long *) &lock->owner;
72 if (rt_mutex_has_waiters(lock))
76 * The rbtree has no waiters enqueued, now make sure that the
77 * lock->owner still has the waiters bit set, otherwise the
78 * following can happen:
84 * l->owner = T1 | HAS_WAITERS;
92 * l->owner = T1 | HAS_WAITERS;
97 * signal(->T2) signal(->T3)
104 * ==> wait list is empty
108 * fixup_rt_mutex_waiters()
109 * if (wait_list_empty(l) {
111 * owner = l->owner & ~HAS_WAITERS;
115 * rt_mutex_unlock(l) fixup_rt_mutex_waiters()
116 * if (wait_list_empty(l) {
117 * owner = l->owner & ~HAS_WAITERS;
118 * cmpxchg(l->owner, T1, NULL)
119 * ===> Success (l->owner = NULL)
125 * With the check for the waiter bit in place T3 on CPU2 will not
126 * overwrite. All tasks fiddling with the waiters bit are
127 * serialized by l->lock, so nothing else can modify the waiters
128 * bit. If the bit is set then nothing can change l->owner either
129 * so the simple RMW is safe. The cmpxchg() will simply fail if it
130 * happens in the middle of the RMW because the waiters bit is
133 owner = READ_ONCE(*p);
134 if (owner & RT_MUTEX_HAS_WAITERS)
135 WRITE_ONCE(*p, owner & ~RT_MUTEX_HAS_WAITERS);
139 * We can speed up the acquire/release, if there's no debugging state to be
142 #ifndef CONFIG_DEBUG_RT_MUTEXES
143 # define rt_mutex_cmpxchg_relaxed(l,c,n) (cmpxchg_relaxed(&l->owner, c, n) == c)
144 # define rt_mutex_cmpxchg_acquire(l,c,n) (cmpxchg_acquire(&l->owner, c, n) == c)
145 # define rt_mutex_cmpxchg_release(l,c,n) (cmpxchg_release(&l->owner, c, n) == c)
148 * Callers must hold the ->wait_lock -- which is the whole purpose as we force
149 * all future threads that attempt to [Rmw] the lock to the slowpath. As such
150 * relaxed semantics suffice.
152 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
154 unsigned long owner, *p = (unsigned long *) &lock->owner;
158 } while (cmpxchg_relaxed(p, owner,
159 owner | RT_MUTEX_HAS_WAITERS) != owner);
163 * Safe fastpath aware unlock:
164 * 1) Clear the waiters bit
165 * 2) Drop lock->wait_lock
166 * 3) Try to unlock the lock with cmpxchg
168 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
170 __releases(lock->wait_lock)
172 struct task_struct *owner = rt_mutex_owner(lock);
174 clear_rt_mutex_waiters(lock);
175 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
177 * If a new waiter comes in between the unlock and the cmpxchg
178 * we have two situations:
182 * cmpxchg(p, owner, 0) == owner
183 * mark_rt_mutex_waiters(lock);
189 * mark_rt_mutex_waiters(lock);
191 * cmpxchg(p, owner, 0) != owner
200 return rt_mutex_cmpxchg_release(lock, owner, NULL);
204 # define rt_mutex_cmpxchg_relaxed(l,c,n) (0)
205 # define rt_mutex_cmpxchg_acquire(l,c,n) (0)
206 # define rt_mutex_cmpxchg_release(l,c,n) (0)
208 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
210 lock->owner = (struct task_struct *)
211 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
215 * Simple slow path only version: lock->owner is protected by lock->wait_lock.
217 static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
219 __releases(lock->wait_lock)
222 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
228 * Only use with rt_mutex_waiter_{less,equal}()
230 #define task_to_waiter(p) \
231 &(struct rt_mutex_waiter){ .prio = (p)->prio, .deadline = (p)->dl.deadline }
234 rt_mutex_waiter_less(struct rt_mutex_waiter *left,
235 struct rt_mutex_waiter *right)
237 if (left->prio < right->prio)
241 * If both waiters have dl_prio(), we check the deadlines of the
243 * If left waiter has a dl_prio(), and we didn't return 1 above,
244 * then right waiter has a dl_prio() too.
246 if (dl_prio(left->prio))
247 return dl_time_before(left->deadline, right->deadline);
253 rt_mutex_waiter_equal(struct rt_mutex_waiter *left,
254 struct rt_mutex_waiter *right)
256 if (left->prio != right->prio)
260 * If both waiters have dl_prio(), we check the deadlines of the
262 * If left waiter has a dl_prio(), and we didn't return 0 above,
263 * then right waiter has a dl_prio() too.
265 if (dl_prio(left->prio))
266 return left->deadline == right->deadline;
272 rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
274 struct rb_node **link = &lock->waiters.rb_node;
275 struct rb_node *parent = NULL;
276 struct rt_mutex_waiter *entry;
281 entry = rb_entry(parent, struct rt_mutex_waiter, tree_entry);
282 if (rt_mutex_waiter_less(waiter, entry)) {
283 link = &parent->rb_left;
285 link = &parent->rb_right;
291 lock->waiters_leftmost = &waiter->tree_entry;
293 rb_link_node(&waiter->tree_entry, parent, link);
294 rb_insert_color(&waiter->tree_entry, &lock->waiters);
298 rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
300 if (RB_EMPTY_NODE(&waiter->tree_entry))
303 if (lock->waiters_leftmost == &waiter->tree_entry)
304 lock->waiters_leftmost = rb_next(&waiter->tree_entry);
306 rb_erase(&waiter->tree_entry, &lock->waiters);
307 RB_CLEAR_NODE(&waiter->tree_entry);
311 rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
313 struct rb_node **link = &task->pi_waiters.rb_node;
314 struct rb_node *parent = NULL;
315 struct rt_mutex_waiter *entry;
320 entry = rb_entry(parent, struct rt_mutex_waiter, pi_tree_entry);
321 if (rt_mutex_waiter_less(waiter, entry)) {
322 link = &parent->rb_left;
324 link = &parent->rb_right;
330 task->pi_waiters_leftmost = &waiter->pi_tree_entry;
332 rb_link_node(&waiter->pi_tree_entry, parent, link);
333 rb_insert_color(&waiter->pi_tree_entry, &task->pi_waiters);
337 rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
339 if (RB_EMPTY_NODE(&waiter->pi_tree_entry))
342 if (task->pi_waiters_leftmost == &waiter->pi_tree_entry)
343 task->pi_waiters_leftmost = rb_next(&waiter->pi_tree_entry);
345 rb_erase(&waiter->pi_tree_entry, &task->pi_waiters);
346 RB_CLEAR_NODE(&waiter->pi_tree_entry);
349 static void rt_mutex_adjust_prio(struct task_struct *p)
351 struct task_struct *pi_task = NULL;
353 lockdep_assert_held(&p->pi_lock);
355 if (task_has_pi_waiters(p))
356 pi_task = task_top_pi_waiter(p)->task;
358 rt_mutex_setprio(p, pi_task);
362 * Deadlock detection is conditional:
364 * If CONFIG_DEBUG_RT_MUTEXES=n, deadlock detection is only conducted
365 * if the detect argument is == RT_MUTEX_FULL_CHAINWALK.
367 * If CONFIG_DEBUG_RT_MUTEXES=y, deadlock detection is always
368 * conducted independent of the detect argument.
370 * If the waiter argument is NULL this indicates the deboost path and
371 * deadlock detection is disabled independent of the detect argument
372 * and the config settings.
374 static bool rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,
375 enum rtmutex_chainwalk chwalk)
378 * This is just a wrapper function for the following call,
379 * because debug_rt_mutex_detect_deadlock() smells like a magic
380 * debug feature and I wanted to keep the cond function in the
381 * main source file along with the comments instead of having
382 * two of the same in the headers.
384 return debug_rt_mutex_detect_deadlock(waiter, chwalk);
388 * Max number of times we'll walk the boosting chain:
390 int max_lock_depth = 1024;
392 static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
394 return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;
398 * Adjust the priority chain. Also used for deadlock detection.
399 * Decreases task's usage by one - may thus free the task.
401 * @task: the task owning the mutex (owner) for which a chain walk is
403 * @chwalk: do we have to carry out deadlock detection?
404 * @orig_lock: the mutex (can be NULL if we are walking the chain to recheck
405 * things for a task that has just got its priority adjusted, and
406 * is waiting on a mutex)
407 * @next_lock: the mutex on which the owner of @orig_lock was blocked before
408 * we dropped its pi_lock. Is never dereferenced, only used for
409 * comparison to detect lock chain changes.
410 * @orig_waiter: rt_mutex_waiter struct for the task that has just donated
411 * its priority to the mutex owner (can be NULL in the case
412 * depicted above or if the top waiter is gone away and we are
413 * actually deboosting the owner)
414 * @top_task: the current top waiter
416 * Returns 0 or -EDEADLK.
418 * Chain walk basics and protection scope
420 * [R] refcount on task
421 * [P] task->pi_lock held
422 * [L] rtmutex->wait_lock held
424 * Step Description Protected by
425 * function arguments:
427 * @orig_lock if != NULL @top_task is blocked on it
428 * @next_lock Unprotected. Cannot be
429 * dereferenced. Only used for
431 * @orig_waiter if != NULL @top_task is blocked on it
432 * @top_task current, or in case of proxy
433 * locking protected by calling
436 * loop_sanity_check();
438 * [1] lock(task->pi_lock); [R] acquire [P]
439 * [2] waiter = task->pi_blocked_on; [P]
440 * [3] check_exit_conditions_1(); [P]
441 * [4] lock = waiter->lock; [P]
442 * [5] if (!try_lock(lock->wait_lock)) { [P] try to acquire [L]
443 * unlock(task->pi_lock); release [P]
446 * [6] check_exit_conditions_2(); [P] + [L]
447 * [7] requeue_lock_waiter(lock, waiter); [P] + [L]
448 * [8] unlock(task->pi_lock); release [P]
449 * put_task_struct(task); release [R]
450 * [9] check_exit_conditions_3(); [L]
451 * [10] task = owner(lock); [L]
452 * get_task_struct(task); [L] acquire [R]
453 * lock(task->pi_lock); [L] acquire [P]
454 * [11] requeue_pi_waiter(tsk, waiters(lock));[P] + [L]
455 * [12] check_exit_conditions_4(); [P] + [L]
456 * [13] unlock(task->pi_lock); release [P]
457 * unlock(lock->wait_lock); release [L]
460 static int rt_mutex_adjust_prio_chain(struct task_struct *task,
461 enum rtmutex_chainwalk chwalk,
462 struct rt_mutex *orig_lock,
463 struct rt_mutex *next_lock,
464 struct rt_mutex_waiter *orig_waiter,
465 struct task_struct *top_task)
467 struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
468 struct rt_mutex_waiter *prerequeue_top_waiter;
469 int ret = 0, depth = 0;
470 struct rt_mutex *lock;
471 bool detect_deadlock;
474 detect_deadlock = rt_mutex_cond_detect_deadlock(orig_waiter, chwalk);
477 * The (de)boosting is a step by step approach with a lot of
478 * pitfalls. We want this to be preemptible and we want hold a
479 * maximum of two locks per step. So we have to check
480 * carefully whether things change under us.
484 * We limit the lock chain length for each invocation.
486 if (++depth > max_lock_depth) {
490 * Print this only once. If the admin changes the limit,
491 * print a new message when reaching the limit again.
493 if (prev_max != max_lock_depth) {
494 prev_max = max_lock_depth;
495 printk(KERN_WARNING "Maximum lock depth %d reached "
496 "task: %s (%d)\n", max_lock_depth,
497 top_task->comm, task_pid_nr(top_task));
499 put_task_struct(task);
505 * We are fully preemptible here and only hold the refcount on
506 * @task. So everything can have changed under us since the
507 * caller or our own code below (goto retry/again) dropped all
512 * [1] Task cannot go away as we did a get_task() before !
514 raw_spin_lock_irq(&task->pi_lock);
517 * [2] Get the waiter on which @task is blocked on.
519 waiter = task->pi_blocked_on;
522 * [3] check_exit_conditions_1() protected by task->pi_lock.
526 * Check whether the end of the boosting chain has been
527 * reached or the state of the chain has changed while we
534 * Check the orig_waiter state. After we dropped the locks,
535 * the previous owner of the lock might have released the lock.
537 if (orig_waiter && !rt_mutex_owner(orig_lock))
541 * We dropped all locks after taking a refcount on @task, so
542 * the task might have moved on in the lock chain or even left
543 * the chain completely and blocks now on an unrelated lock or
546 * We stored the lock on which @task was blocked in @next_lock,
547 * so we can detect the chain change.
549 if (next_lock != waiter->lock)
553 * Drop out, when the task has no waiters. Note,
554 * top_waiter can be NULL, when we are in the deboosting
558 if (!task_has_pi_waiters(task))
561 * If deadlock detection is off, we stop here if we
562 * are not the top pi waiter of the task. If deadlock
563 * detection is enabled we continue, but stop the
564 * requeueing in the chain walk.
566 if (top_waiter != task_top_pi_waiter(task)) {
567 if (!detect_deadlock)
575 * If the waiter priority is the same as the task priority
576 * then there is no further priority adjustment necessary. If
577 * deadlock detection is off, we stop the chain walk. If its
578 * enabled we continue, but stop the requeueing in the chain
581 if (rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {
582 if (!detect_deadlock)
589 * [4] Get the next lock
593 * [5] We need to trylock here as we are holding task->pi_lock,
594 * which is the reverse lock order versus the other rtmutex
597 if (!raw_spin_trylock(&lock->wait_lock)) {
598 raw_spin_unlock_irq(&task->pi_lock);
604 * [6] check_exit_conditions_2() protected by task->pi_lock and
607 * Deadlock detection. If the lock is the same as the original
608 * lock which caused us to walk the lock chain or if the
609 * current lock is owned by the task which initiated the chain
610 * walk, we detected a deadlock.
612 if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
613 debug_rt_mutex_deadlock(chwalk, orig_waiter, lock);
614 raw_spin_unlock(&lock->wait_lock);
620 * If we just follow the lock chain for deadlock detection, no
621 * need to do all the requeue operations. To avoid a truckload
622 * of conditionals around the various places below, just do the
623 * minimum chain walk checks.
627 * No requeue[7] here. Just release @task [8]
629 raw_spin_unlock(&task->pi_lock);
630 put_task_struct(task);
633 * [9] check_exit_conditions_3 protected by lock->wait_lock.
634 * If there is no owner of the lock, end of chain.
636 if (!rt_mutex_owner(lock)) {
637 raw_spin_unlock_irq(&lock->wait_lock);
641 /* [10] Grab the next task, i.e. owner of @lock */
642 task = rt_mutex_owner(lock);
643 get_task_struct(task);
644 raw_spin_lock(&task->pi_lock);
647 * No requeue [11] here. We just do deadlock detection.
649 * [12] Store whether owner is blocked
650 * itself. Decision is made after dropping the locks
652 next_lock = task_blocked_on_lock(task);
654 * Get the top waiter for the next iteration
656 top_waiter = rt_mutex_top_waiter(lock);
658 /* [13] Drop locks */
659 raw_spin_unlock(&task->pi_lock);
660 raw_spin_unlock_irq(&lock->wait_lock);
662 /* If owner is not blocked, end of chain. */
669 * Store the current top waiter before doing the requeue
670 * operation on @lock. We need it for the boost/deboost
673 prerequeue_top_waiter = rt_mutex_top_waiter(lock);
675 /* [7] Requeue the waiter in the lock waiter tree. */
676 rt_mutex_dequeue(lock, waiter);
679 * Update the waiter prio fields now that we're dequeued.
681 * These values can have changed through either:
683 * sys_sched_set_scheduler() / sys_sched_setattr()
687 * DL CBS enforcement advancing the effective deadline.
689 * Even though pi_waiters also uses these fields, and that tree is only
690 * updated in [11], we can do this here, since we hold [L], which
691 * serializes all pi_waiters access and rb_erase() does not care about
692 * the values of the node being removed.
694 waiter->prio = task->prio;
695 waiter->deadline = task->dl.deadline;
697 rt_mutex_enqueue(lock, waiter);
699 /* [8] Release the task */
700 raw_spin_unlock(&task->pi_lock);
701 put_task_struct(task);
704 * [9] check_exit_conditions_3 protected by lock->wait_lock.
706 * We must abort the chain walk if there is no lock owner even
707 * in the dead lock detection case, as we have nothing to
708 * follow here. This is the end of the chain we are walking.
710 if (!rt_mutex_owner(lock)) {
712 * If the requeue [7] above changed the top waiter,
713 * then we need to wake the new top waiter up to try
716 if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))
717 wake_up_process(rt_mutex_top_waiter(lock)->task);
718 raw_spin_unlock_irq(&lock->wait_lock);
722 /* [10] Grab the next task, i.e. the owner of @lock */
723 task = rt_mutex_owner(lock);
724 get_task_struct(task);
725 raw_spin_lock(&task->pi_lock);
727 /* [11] requeue the pi waiters if necessary */
728 if (waiter == rt_mutex_top_waiter(lock)) {
730 * The waiter became the new top (highest priority)
731 * waiter on the lock. Replace the previous top waiter
732 * in the owner tasks pi waiters tree with this waiter
733 * and adjust the priority of the owner.
735 rt_mutex_dequeue_pi(task, prerequeue_top_waiter);
736 rt_mutex_enqueue_pi(task, waiter);
737 rt_mutex_adjust_prio(task);
739 } else if (prerequeue_top_waiter == waiter) {
741 * The waiter was the top waiter on the lock, but is
742 * no longer the top prority waiter. Replace waiter in
743 * the owner tasks pi waiters tree with the new top
744 * (highest priority) waiter and adjust the priority
746 * The new top waiter is stored in @waiter so that
747 * @waiter == @top_waiter evaluates to true below and
748 * we continue to deboost the rest of the chain.
750 rt_mutex_dequeue_pi(task, waiter);
751 waiter = rt_mutex_top_waiter(lock);
752 rt_mutex_enqueue_pi(task, waiter);
753 rt_mutex_adjust_prio(task);
756 * Nothing changed. No need to do any priority
762 * [12] check_exit_conditions_4() protected by task->pi_lock
763 * and lock->wait_lock. The actual decisions are made after we
766 * Check whether the task which owns the current lock is pi
767 * blocked itself. If yes we store a pointer to the lock for
768 * the lock chain change detection above. After we dropped
769 * task->pi_lock next_lock cannot be dereferenced anymore.
771 next_lock = task_blocked_on_lock(task);
773 * Store the top waiter of @lock for the end of chain walk
776 top_waiter = rt_mutex_top_waiter(lock);
778 /* [13] Drop the locks */
779 raw_spin_unlock(&task->pi_lock);
780 raw_spin_unlock_irq(&lock->wait_lock);
783 * Make the actual exit decisions [12], based on the stored
786 * We reached the end of the lock chain. Stop right here. No
787 * point to go back just to figure that out.
793 * If the current waiter is not the top waiter on the lock,
794 * then we can stop the chain walk here if we are not in full
795 * deadlock detection mode.
797 if (!detect_deadlock && waiter != top_waiter)
803 raw_spin_unlock_irq(&task->pi_lock);
805 put_task_struct(task);
811 * Try to take an rt-mutex
813 * Must be called with lock->wait_lock held and interrupts disabled
815 * @lock: The lock to be acquired.
816 * @task: The task which wants to acquire the lock
817 * @waiter: The waiter that is queued to the lock's wait tree if the
818 * callsite called task_blocked_on_lock(), otherwise NULL
820 static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
821 struct rt_mutex_waiter *waiter)
823 lockdep_assert_held(&lock->wait_lock);
826 * Before testing whether we can acquire @lock, we set the
827 * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all
828 * other tasks which try to modify @lock into the slow path
829 * and they serialize on @lock->wait_lock.
831 * The RT_MUTEX_HAS_WAITERS bit can have a transitional state
832 * as explained at the top of this file if and only if:
834 * - There is a lock owner. The caller must fixup the
835 * transient state if it does a trylock or leaves the lock
836 * function due to a signal or timeout.
838 * - @task acquires the lock and there are no other
839 * waiters. This is undone in rt_mutex_set_owner(@task) at
840 * the end of this function.
842 mark_rt_mutex_waiters(lock);
845 * If @lock has an owner, give up.
847 if (rt_mutex_owner(lock))
851 * If @waiter != NULL, @task has already enqueued the waiter
852 * into @lock waiter tree. If @waiter == NULL then this is a
857 * If waiter is not the highest priority waiter of
860 if (waiter != rt_mutex_top_waiter(lock))
864 * We can acquire the lock. Remove the waiter from the
867 rt_mutex_dequeue(lock, waiter);
871 * If the lock has waiters already we check whether @task is
872 * eligible to take over the lock.
874 * If there are no other waiters, @task can acquire
875 * the lock. @task->pi_blocked_on is NULL, so it does
876 * not need to be dequeued.
878 if (rt_mutex_has_waiters(lock)) {
880 * If @task->prio is greater than or equal to
881 * the top waiter priority (kernel view),
884 if (!rt_mutex_waiter_less(task_to_waiter(task),
885 rt_mutex_top_waiter(lock)))
889 * The current top waiter stays enqueued. We
890 * don't have to change anything in the lock
895 * No waiters. Take the lock without the
896 * pi_lock dance.@task->pi_blocked_on is NULL
897 * and we have no waiters to enqueue in @task
905 * Clear @task->pi_blocked_on. Requires protection by
906 * @task->pi_lock. Redundant operation for the @waiter == NULL
907 * case, but conditionals are more expensive than a redundant
910 raw_spin_lock(&task->pi_lock);
911 task->pi_blocked_on = NULL;
913 * Finish the lock acquisition. @task is the new owner. If
914 * other waiters exist we have to insert the highest priority
915 * waiter into @task->pi_waiters tree.
917 if (rt_mutex_has_waiters(lock))
918 rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));
919 raw_spin_unlock(&task->pi_lock);
922 /* We got the lock. */
923 debug_rt_mutex_lock(lock);
926 * This either preserves the RT_MUTEX_HAS_WAITERS bit if there
927 * are still waiters or clears it.
929 rt_mutex_set_owner(lock, task);
935 * Task blocks on lock.
937 * Prepare waiter and propagate pi chain
939 * This must be called with lock->wait_lock held and interrupts disabled
941 static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
942 struct rt_mutex_waiter *waiter,
943 struct task_struct *task,
944 enum rtmutex_chainwalk chwalk)
946 struct task_struct *owner = rt_mutex_owner(lock);
947 struct rt_mutex_waiter *top_waiter = waiter;
948 struct rt_mutex *next_lock;
949 int chain_walk = 0, res;
951 lockdep_assert_held(&lock->wait_lock);
954 * Early deadlock detection. We really don't want the task to
955 * enqueue on itself just to untangle the mess later. It's not
956 * only an optimization. We drop the locks, so another waiter
957 * can come in before the chain walk detects the deadlock. So
958 * the other will detect the deadlock and return -EDEADLOCK,
959 * which is wrong, as the other waiter is not in a deadlock
965 raw_spin_lock(&task->pi_lock);
966 rt_mutex_adjust_prio(task);
969 waiter->prio = task->prio;
970 waiter->deadline = task->dl.deadline;
972 /* Get the top priority waiter on the lock */
973 if (rt_mutex_has_waiters(lock))
974 top_waiter = rt_mutex_top_waiter(lock);
975 rt_mutex_enqueue(lock, waiter);
977 task->pi_blocked_on = waiter;
979 raw_spin_unlock(&task->pi_lock);
984 raw_spin_lock(&owner->pi_lock);
985 if (waiter == rt_mutex_top_waiter(lock)) {
986 rt_mutex_dequeue_pi(owner, top_waiter);
987 rt_mutex_enqueue_pi(owner, waiter);
989 rt_mutex_adjust_prio(owner);
990 if (owner->pi_blocked_on)
992 } else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) {
996 /* Store the lock on which owner is blocked or NULL */
997 next_lock = task_blocked_on_lock(owner);
999 raw_spin_unlock(&owner->pi_lock);
1001 * Even if full deadlock detection is on, if the owner is not
1002 * blocked itself, we can avoid finding this out in the chain
1005 if (!chain_walk || !next_lock)
1009 * The owner can't disappear while holding a lock,
1010 * so the owner struct is protected by wait_lock.
1011 * Gets dropped in rt_mutex_adjust_prio_chain()!
1013 get_task_struct(owner);
1015 raw_spin_unlock_irq(&lock->wait_lock);
1017 res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,
1018 next_lock, waiter, task);
1020 raw_spin_lock_irq(&lock->wait_lock);
1026 * Remove the top waiter from the current tasks pi waiter tree and
1029 * Called with lock->wait_lock held and interrupts disabled.
1031 static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
1032 struct rt_mutex *lock)
1034 struct rt_mutex_waiter *waiter;
1036 raw_spin_lock(¤t->pi_lock);
1038 waiter = rt_mutex_top_waiter(lock);
1041 * Remove it from current->pi_waiters and deboost.
1043 * We must in fact deboost here in order to ensure we call
1044 * rt_mutex_setprio() to update p->pi_top_task before the
1047 rt_mutex_dequeue_pi(current, waiter);
1048 rt_mutex_adjust_prio(current);
1051 * As we are waking up the top waiter, and the waiter stays
1052 * queued on the lock until it gets the lock, this lock
1053 * obviously has waiters. Just set the bit here and this has
1054 * the added benefit of forcing all new tasks into the
1055 * slow path making sure no task of lower priority than
1056 * the top waiter can steal this lock.
1058 lock->owner = (void *) RT_MUTEX_HAS_WAITERS;
1061 * We deboosted before waking the top waiter task such that we don't
1062 * run two tasks with the 'same' priority (and ensure the
1063 * p->pi_top_task pointer points to a blocked task). This however can
1064 * lead to priority inversion if we would get preempted after the
1065 * deboost but before waking our donor task, hence the preempt_disable()
1068 * Pairs with preempt_enable() in rt_mutex_postunlock();
1071 wake_q_add(wake_q, waiter->task);
1072 raw_spin_unlock(¤t->pi_lock);
1076 * Remove a waiter from a lock and give up
1078 * Must be called with lock->wait_lock held and interrupts disabled. I must
1079 * have just failed to try_to_take_rt_mutex().
1081 static void remove_waiter(struct rt_mutex *lock,
1082 struct rt_mutex_waiter *waiter)
1084 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
1085 struct task_struct *owner = rt_mutex_owner(lock);
1086 struct rt_mutex *next_lock;
1088 lockdep_assert_held(&lock->wait_lock);
1090 raw_spin_lock(¤t->pi_lock);
1091 rt_mutex_dequeue(lock, waiter);
1092 current->pi_blocked_on = NULL;
1093 raw_spin_unlock(¤t->pi_lock);
1096 * Only update priority if the waiter was the highest priority
1097 * waiter of the lock and there is an owner to update.
1099 if (!owner || !is_top_waiter)
1102 raw_spin_lock(&owner->pi_lock);
1104 rt_mutex_dequeue_pi(owner, waiter);
1106 if (rt_mutex_has_waiters(lock))
1107 rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));
1109 rt_mutex_adjust_prio(owner);
1111 /* Store the lock on which owner is blocked or NULL */
1112 next_lock = task_blocked_on_lock(owner);
1114 raw_spin_unlock(&owner->pi_lock);
1117 * Don't walk the chain, if the owner task is not blocked
1123 /* gets dropped in rt_mutex_adjust_prio_chain()! */
1124 get_task_struct(owner);
1126 raw_spin_unlock_irq(&lock->wait_lock);
1128 rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,
1129 next_lock, NULL, current);
1131 raw_spin_lock_irq(&lock->wait_lock);
1135 * Recheck the pi chain, in case we got a priority setting
1137 * Called from sched_setscheduler
1139 void rt_mutex_adjust_pi(struct task_struct *task)
1141 struct rt_mutex_waiter *waiter;
1142 struct rt_mutex *next_lock;
1143 unsigned long flags;
1145 raw_spin_lock_irqsave(&task->pi_lock, flags);
1147 waiter = task->pi_blocked_on;
1148 if (!waiter || rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {
1149 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
1152 next_lock = waiter->lock;
1153 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
1155 /* gets dropped in rt_mutex_adjust_prio_chain()! */
1156 get_task_struct(task);
1158 rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL,
1159 next_lock, NULL, task);
1162 void rt_mutex_init_waiter(struct rt_mutex_waiter *waiter)
1164 debug_rt_mutex_init_waiter(waiter);
1165 RB_CLEAR_NODE(&waiter->pi_tree_entry);
1166 RB_CLEAR_NODE(&waiter->tree_entry);
1167 waiter->task = NULL;
1171 * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
1172 * @lock: the rt_mutex to take
1173 * @state: the state the task should block in (TASK_INTERRUPTIBLE
1174 * or TASK_UNINTERRUPTIBLE)
1175 * @timeout: the pre-initialized and started timer, or NULL for none
1176 * @waiter: the pre-initialized rt_mutex_waiter
1178 * Must be called with lock->wait_lock held and interrupts disabled
1181 __rt_mutex_slowlock(struct rt_mutex *lock, int state,
1182 struct hrtimer_sleeper *timeout,
1183 struct rt_mutex_waiter *waiter)
1188 /* Try to acquire the lock: */
1189 if (try_to_take_rt_mutex(lock, current, waiter))
1193 * TASK_INTERRUPTIBLE checks for signals and
1194 * timeout. Ignored otherwise.
1196 if (likely(state == TASK_INTERRUPTIBLE)) {
1197 /* Signal pending? */
1198 if (signal_pending(current))
1200 if (timeout && !timeout->task)
1206 raw_spin_unlock_irq(&lock->wait_lock);
1208 debug_rt_mutex_print_deadlock(waiter);
1212 raw_spin_lock_irq(&lock->wait_lock);
1213 set_current_state(state);
1216 __set_current_state(TASK_RUNNING);
1220 static void rt_mutex_handle_deadlock(int res, int detect_deadlock,
1221 struct rt_mutex_waiter *w)
1224 * If the result is not -EDEADLOCK or the caller requested
1225 * deadlock detection, nothing to do here.
1227 if (res != -EDEADLOCK || detect_deadlock)
1231 * Yell lowdly and stop the task right here.
1233 rt_mutex_print_deadlock(w);
1235 set_current_state(TASK_INTERRUPTIBLE);
1241 * Slow path lock function:
1244 rt_mutex_slowlock(struct rt_mutex *lock, int state,
1245 struct hrtimer_sleeper *timeout,
1246 enum rtmutex_chainwalk chwalk)
1248 struct rt_mutex_waiter waiter;
1249 unsigned long flags;
1252 rt_mutex_init_waiter(&waiter);
1255 * Technically we could use raw_spin_[un]lock_irq() here, but this can
1256 * be called in early boot if the cmpxchg() fast path is disabled
1257 * (debug, no architecture support). In this case we will acquire the
1258 * rtmutex with lock->wait_lock held. But we cannot unconditionally
1259 * enable interrupts in that early boot case. So we need to use the
1260 * irqsave/restore variants.
1262 raw_spin_lock_irqsave(&lock->wait_lock, flags);
1264 /* Try to acquire the lock again: */
1265 if (try_to_take_rt_mutex(lock, current, NULL)) {
1266 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1270 set_current_state(state);
1272 /* Setup the timer, when timeout != NULL */
1273 if (unlikely(timeout))
1274 hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
1276 ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk);
1279 /* sleep on the mutex */
1280 ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
1282 if (unlikely(ret)) {
1283 __set_current_state(TASK_RUNNING);
1284 if (rt_mutex_has_waiters(lock))
1285 remove_waiter(lock, &waiter);
1286 rt_mutex_handle_deadlock(ret, chwalk, &waiter);
1290 * try_to_take_rt_mutex() sets the waiter bit
1291 * unconditionally. We might have to fix that up.
1293 fixup_rt_mutex_waiters(lock);
1295 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1297 /* Remove pending timer: */
1298 if (unlikely(timeout))
1299 hrtimer_cancel(&timeout->timer);
1301 debug_rt_mutex_free_waiter(&waiter);
1307 * Slow path try-lock function:
1309 static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)
1311 unsigned long flags;
1315 * If the lock already has an owner we fail to get the lock.
1316 * This can be done without taking the @lock->wait_lock as
1317 * it is only being read, and this is a trylock anyway.
1319 if (rt_mutex_owner(lock))
1323 * The mutex has currently no owner. Lock the wait lock and try to
1324 * acquire the lock. We use irqsave here to support early boot calls.
1326 raw_spin_lock_irqsave(&lock->wait_lock, flags);
1328 ret = try_to_take_rt_mutex(lock, current, NULL);
1331 * try_to_take_rt_mutex() sets the lock waiters bit
1332 * unconditionally. Clean this up.
1334 fixup_rt_mutex_waiters(lock);
1336 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1342 * Slow path to release a rt-mutex.
1344 * Return whether the current task needs to call rt_mutex_postunlock().
1346 static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock,
1347 struct wake_q_head *wake_q)
1349 unsigned long flags;
1351 /* irqsave required to support early boot calls */
1352 raw_spin_lock_irqsave(&lock->wait_lock, flags);
1354 debug_rt_mutex_unlock(lock);
1357 * We must be careful here if the fast path is enabled. If we
1358 * have no waiters queued we cannot set owner to NULL here
1361 * foo->lock->owner = NULL;
1362 * rtmutex_lock(foo->lock); <- fast path
1363 * free = atomic_dec_and_test(foo->refcnt);
1364 * rtmutex_unlock(foo->lock); <- fast path
1367 * raw_spin_unlock(foo->lock->wait_lock);
1369 * So for the fastpath enabled kernel:
1371 * Nothing can set the waiters bit as long as we hold
1372 * lock->wait_lock. So we do the following sequence:
1374 * owner = rt_mutex_owner(lock);
1375 * clear_rt_mutex_waiters(lock);
1376 * raw_spin_unlock(&lock->wait_lock);
1377 * if (cmpxchg(&lock->owner, owner, 0) == owner)
1381 * The fastpath disabled variant is simple as all access to
1382 * lock->owner is serialized by lock->wait_lock:
1384 * lock->owner = NULL;
1385 * raw_spin_unlock(&lock->wait_lock);
1387 while (!rt_mutex_has_waiters(lock)) {
1388 /* Drops lock->wait_lock ! */
1389 if (unlock_rt_mutex_safe(lock, flags) == true)
1391 /* Relock the rtmutex and try again */
1392 raw_spin_lock_irqsave(&lock->wait_lock, flags);
1396 * The wakeup next waiter path does not suffer from the above
1397 * race. See the comments there.
1399 * Queue the next waiter for wakeup once we release the wait_lock.
1401 mark_wakeup_next_waiter(wake_q, lock);
1402 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1404 return true; /* call rt_mutex_postunlock() */
1408 * debug aware fast / slowpath lock,trylock,unlock
1410 * The atomic acquire/release ops are compiled away, when either the
1411 * architecture does not support cmpxchg or when debugging is enabled.
1414 rt_mutex_fastlock(struct rt_mutex *lock, int state,
1415 int (*slowfn)(struct rt_mutex *lock, int state,
1416 struct hrtimer_sleeper *timeout,
1417 enum rtmutex_chainwalk chwalk))
1419 if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
1422 return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);
1426 rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
1427 struct hrtimer_sleeper *timeout,
1428 enum rtmutex_chainwalk chwalk,
1429 int (*slowfn)(struct rt_mutex *lock, int state,
1430 struct hrtimer_sleeper *timeout,
1431 enum rtmutex_chainwalk chwalk))
1433 if (chwalk == RT_MUTEX_MIN_CHAINWALK &&
1434 likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
1437 return slowfn(lock, state, timeout, chwalk);
1441 rt_mutex_fasttrylock(struct rt_mutex *lock,
1442 int (*slowfn)(struct rt_mutex *lock))
1444 if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
1447 return slowfn(lock);
1451 * Performs the wakeup of the the top-waiter and re-enables preemption.
1453 void rt_mutex_postunlock(struct wake_q_head *wake_q)
1457 /* Pairs with preempt_disable() in rt_mutex_slowunlock() */
1462 rt_mutex_fastunlock(struct rt_mutex *lock,
1463 bool (*slowfn)(struct rt_mutex *lock,
1464 struct wake_q_head *wqh))
1466 DEFINE_WAKE_Q(wake_q);
1468 if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
1471 if (slowfn(lock, &wake_q))
1472 rt_mutex_postunlock(&wake_q);
1476 * rt_mutex_lock - lock a rt_mutex
1478 * @lock: the rt_mutex to be locked
1480 void __sched rt_mutex_lock(struct rt_mutex *lock)
1484 mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1485 rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock);
1487 EXPORT_SYMBOL_GPL(rt_mutex_lock);
1490 * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
1492 * @lock: the rt_mutex to be locked
1496 * -EINTR when interrupted by a signal
1498 int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)
1504 mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1505 ret = rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock);
1507 mutex_release(&lock->dep_map, 1, _RET_IP_);
1511 EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
1514 * Futex variant, must not use fastpath.
1516 int __sched rt_mutex_futex_trylock(struct rt_mutex *lock)
1518 return rt_mutex_slowtrylock(lock);
1522 * rt_mutex_timed_lock - lock a rt_mutex interruptible
1523 * the timeout structure is provided
1526 * @lock: the rt_mutex to be locked
1527 * @timeout: timeout structure or NULL (no timeout)
1531 * -EINTR when interrupted by a signal
1532 * -ETIMEDOUT when the timeout expired
1535 rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout)
1541 mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
1542 ret = rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
1543 RT_MUTEX_MIN_CHAINWALK,
1546 mutex_release(&lock->dep_map, 1, _RET_IP_);
1550 EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
1553 * rt_mutex_trylock - try to lock a rt_mutex
1555 * @lock: the rt_mutex to be locked
1557 * This function can only be called in thread context. It's safe to
1558 * call it from atomic regions, but not from hard interrupt or soft
1559 * interrupt context.
1561 * Returns 1 on success and 0 on contention
1563 int __sched rt_mutex_trylock(struct rt_mutex *lock)
1567 if (WARN_ON_ONCE(in_irq() || in_nmi() || in_serving_softirq()))
1570 ret = rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
1572 mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
1576 EXPORT_SYMBOL_GPL(rt_mutex_trylock);
1579 * rt_mutex_unlock - unlock a rt_mutex
1581 * @lock: the rt_mutex to be unlocked
1583 void __sched rt_mutex_unlock(struct rt_mutex *lock)
1585 mutex_release(&lock->dep_map, 1, _RET_IP_);
1586 rt_mutex_fastunlock(lock, rt_mutex_slowunlock);
1588 EXPORT_SYMBOL_GPL(rt_mutex_unlock);
1591 * Futex variant, that since futex variants do not use the fast-path, can be
1592 * simple and will not need to retry.
1594 bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock,
1595 struct wake_q_head *wake_q)
1597 lockdep_assert_held(&lock->wait_lock);
1599 debug_rt_mutex_unlock(lock);
1601 if (!rt_mutex_has_waiters(lock)) {
1603 return false; /* done */
1607 * We've already deboosted, mark_wakeup_next_waiter() will
1608 * retain preempt_disabled when we drop the wait_lock, to
1609 * avoid inversion prior to the wakeup. preempt_disable()
1610 * therein pairs with rt_mutex_postunlock().
1612 mark_wakeup_next_waiter(wake_q, lock);
1614 return true; /* call postunlock() */
1617 void __sched rt_mutex_futex_unlock(struct rt_mutex *lock)
1619 DEFINE_WAKE_Q(wake_q);
1622 raw_spin_lock_irq(&lock->wait_lock);
1623 postunlock = __rt_mutex_futex_unlock(lock, &wake_q);
1624 raw_spin_unlock_irq(&lock->wait_lock);
1627 rt_mutex_postunlock(&wake_q);
1631 * rt_mutex_destroy - mark a mutex unusable
1632 * @lock: the mutex to be destroyed
1634 * This function marks the mutex uninitialized, and any subsequent
1635 * use of the mutex is forbidden. The mutex must not be locked when
1636 * this function is called.
1638 void rt_mutex_destroy(struct rt_mutex *lock)
1640 WARN_ON(rt_mutex_is_locked(lock));
1641 #ifdef CONFIG_DEBUG_RT_MUTEXES
1645 EXPORT_SYMBOL_GPL(rt_mutex_destroy);
1648 * __rt_mutex_init - initialize the rt lock
1650 * @lock: the rt lock to be initialized
1652 * Initialize the rt lock to unlocked state.
1654 * Initializing of a locked rt lock is not allowed
1656 void __rt_mutex_init(struct rt_mutex *lock, const char *name,
1657 struct lock_class_key *key)
1660 raw_spin_lock_init(&lock->wait_lock);
1661 lock->waiters = RB_ROOT;
1662 lock->waiters_leftmost = NULL;
1665 debug_rt_mutex_init(lock, name, key);
1667 EXPORT_SYMBOL_GPL(__rt_mutex_init);
1670 * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
1673 * @lock: the rt_mutex to be locked
1674 * @proxy_owner:the task to set as owner
1676 * No locking. Caller has to do serializing itself
1678 * Special API call for PI-futex support. This initializes the rtmutex and
1679 * assigns it to @proxy_owner. Concurrent operations on the rtmutex are not
1680 * possible at this point because the pi_state which contains the rtmutex
1681 * is not yet visible to other tasks.
1683 void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
1684 struct task_struct *proxy_owner)
1686 __rt_mutex_init(lock, NULL, NULL);
1687 debug_rt_mutex_proxy_lock(lock, proxy_owner);
1688 rt_mutex_set_owner(lock, proxy_owner);
1692 * rt_mutex_proxy_unlock - release a lock on behalf of owner
1694 * @lock: the rt_mutex to be locked
1696 * No locking. Caller has to do serializing itself
1698 * Special API call for PI-futex support. This merrily cleans up the rtmutex
1699 * (debugging) state. Concurrent operations on this rt_mutex are not
1700 * possible because it belongs to the pi_state which is about to be freed
1701 * and it is not longer visible to other tasks.
1703 void rt_mutex_proxy_unlock(struct rt_mutex *lock,
1704 struct task_struct *proxy_owner)
1706 debug_rt_mutex_proxy_unlock(lock);
1707 rt_mutex_set_owner(lock, NULL);
1710 int __rt_mutex_start_proxy_lock(struct rt_mutex *lock,
1711 struct rt_mutex_waiter *waiter,
1712 struct task_struct *task)
1716 if (try_to_take_rt_mutex(lock, task, NULL))
1719 /* We enforce deadlock detection for futexes */
1720 ret = task_blocks_on_rt_mutex(lock, waiter, task,
1721 RT_MUTEX_FULL_CHAINWALK);
1723 if (ret && !rt_mutex_owner(lock)) {
1725 * Reset the return value. We might have
1726 * returned with -EDEADLK and the owner
1727 * released the lock while we were walking the
1728 * pi chain. Let the waiter sort it out.
1734 remove_waiter(lock, waiter);
1736 debug_rt_mutex_print_deadlock(waiter);
1742 * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
1743 * @lock: the rt_mutex to take
1744 * @waiter: the pre-initialized rt_mutex_waiter
1745 * @task: the task to prepare
1748 * 0 - task blocked on lock
1749 * 1 - acquired the lock for task, caller should wake it up
1752 * Special API call for FUTEX_REQUEUE_PI support.
1754 int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
1755 struct rt_mutex_waiter *waiter,
1756 struct task_struct *task)
1760 raw_spin_lock_irq(&lock->wait_lock);
1761 ret = __rt_mutex_start_proxy_lock(lock, waiter, task);
1762 raw_spin_unlock_irq(&lock->wait_lock);
1768 * rt_mutex_next_owner - return the next owner of the lock
1770 * @lock: the rt lock query
1772 * Returns the next owner of the lock or NULL
1774 * Caller has to serialize against other accessors to the lock
1777 * Special API call for PI-futex support
1779 struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
1781 if (!rt_mutex_has_waiters(lock))
1784 return rt_mutex_top_waiter(lock)->task;
1788 * rt_mutex_wait_proxy_lock() - Wait for lock acquisition
1789 * @lock: the rt_mutex we were woken on
1790 * @to: the timeout, null if none. hrtimer should already have
1792 * @waiter: the pre-initialized rt_mutex_waiter
1794 * Wait for the the lock acquisition started on our behalf by
1795 * rt_mutex_start_proxy_lock(). Upon failure, the caller must call
1796 * rt_mutex_cleanup_proxy_lock().
1800 * <0 - error, one of -EINTR, -ETIMEDOUT
1802 * Special API call for PI-futex support
1804 int rt_mutex_wait_proxy_lock(struct rt_mutex *lock,
1805 struct hrtimer_sleeper *to,
1806 struct rt_mutex_waiter *waiter)
1810 raw_spin_lock_irq(&lock->wait_lock);
1811 /* sleep on the mutex */
1812 set_current_state(TASK_INTERRUPTIBLE);
1813 ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
1815 * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
1816 * have to fix that up.
1818 fixup_rt_mutex_waiters(lock);
1819 raw_spin_unlock_irq(&lock->wait_lock);
1825 * rt_mutex_cleanup_proxy_lock() - Cleanup failed lock acquisition
1826 * @lock: the rt_mutex we were woken on
1827 * @waiter: the pre-initialized rt_mutex_waiter
1829 * Attempt to clean up after a failed rt_mutex_wait_proxy_lock().
1831 * Unless we acquired the lock; we're still enqueued on the wait-list and can
1832 * in fact still be granted ownership until we're removed. Therefore we can
1833 * find we are in fact the owner and must disregard the
1834 * rt_mutex_wait_proxy_lock() failure.
1837 * true - did the cleanup, we done.
1838 * false - we acquired the lock after rt_mutex_wait_proxy_lock() returned,
1839 * caller should disregards its return value.
1841 * Special API call for PI-futex support
1843 bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock,
1844 struct rt_mutex_waiter *waiter)
1846 bool cleanup = false;
1848 raw_spin_lock_irq(&lock->wait_lock);
1850 * Do an unconditional try-lock, this deals with the lock stealing
1851 * state where __rt_mutex_futex_unlock() -> mark_wakeup_next_waiter()
1852 * sets a NULL owner.
1854 * We're not interested in the return value, because the subsequent
1855 * test on rt_mutex_owner() will infer that. If the trylock succeeded,
1856 * we will own the lock and it will have removed the waiter. If we
1857 * failed the trylock, we're still not owner and we need to remove
1860 try_to_take_rt_mutex(lock, current, waiter);
1862 * Unless we're the owner; we're still enqueued on the wait_list.
1863 * So check if we became owner, if not, take us off the wait_list.
1865 if (rt_mutex_owner(lock) != current) {
1866 remove_waiter(lock, waiter);
1870 * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
1871 * have to fix that up.
1873 fixup_rt_mutex_waiters(lock);
1875 raw_spin_unlock_irq(&lock->wait_lock);