// SPDX-License-Identifier: GPL-2.0-only
/*
* RT-Mutexes: simple blocking mutual exclusion locks with PI support
*
* started by Ingo Molnar and Thomas Gleixner.
*
* Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
* Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
* Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
* Copyright (C) 2006 Esben Nielsen
*
* See Documentation/locking/rt-mutex-design.rst for details.
*/
#include <linux/sched.h>
#include <linux/sched/debug.h>
#include <linux/sched/deadline.h>
#include <linux/sched/signal.h>
#include <linux/sched/rt.h>
#include <linux/sched/wake_q.h>
#include "rtmutex_common.h"
/*
* lock->owner state tracking:
*
* lock->owner holds the task_struct pointer of the owner. Bit 0
* is used to keep track of the "lock has waiters" state.
*
* owner bit0
* NULL 0 lock is free (fast acquire possible)
* NULL 1 lock is free and has waiters and the top waiter
* is going to take the lock*
* taskpointer 0 lock is held (fast release possible)
* taskpointer 1 lock is held and has waiters**
*
* The fast atomic compare exchange based acquire and release is only
* possible when bit 0 of lock->owner is 0.
*
* (*) It also can be a transitional state when grabbing the lock
* with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
* we need to set the bit0 before looking at the lock, and the owner may be
* NULL in this small time, hence this can be a transitional state.
*
* (**) There is a small time when bit 0 is set but there are no
* waiters. This can happen when grabbing the lock in the slow path.
* To prevent a cmpxchg of the owner releasing the lock, we need to
* set this bit before looking at the lock.
*/
static __always_inline void
rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)
{
unsigned long val = (unsigned long)owner;
if (rt_mutex_has_waiters(lock))
val |= RT_MUTEX_HAS_WAITERS;
WRITE_ONCE(lock->owner, (struct task_struct *)val);
}
static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock)
{
lock->