kernel/locking/rwsem.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * The least significant 2 bits of the owner value has the following
   4  * meanings when set.
   5  *  - RWSEM_READER_OWNED (bit 0): The rwsem is owned by readers
   6  *  - RWSEM_ANONYMOUSLY_OWNED (bit 1): The rwsem is anonymously owned,
   7  *    i.e. the owner(s) cannot be readily determined. It can be reader
   8  *    owned or the owning writer is indeterminate.
   9  *
  10  * When a writer acquires a rwsem, it puts its task_struct pointer
  11  * into the owner field. It is cleared after an unlock.
  12  *
  13  * When a reader acquires a rwsem, it will also puts its task_struct
  14  * pointer into the owner field with both the RWSEM_READER_OWNED and
  15  * RWSEM_ANONYMOUSLY_OWNED bits set. On unlock, the owner field will
  16  * largely be left untouched. So for a free or reader-owned rwsem,
  17  * the owner value may contain information about the last reader that
  18  * acquires the rwsem. The anonymous bit is set because that particular
  19  * reader may or may not still own the lock.
  20  *
  21  * That information may be helpful in debugging cases where the system
  22  * seems to hang on a reader owned rwsem especially if only one reader
  23  * is involved. Ideally we would like to track all the readers that own
  24  * a rwsem, but the overhead is simply too big.
  25  */
  26 #define RWSEM_READER_OWNED      (1UL << 0)
  27 #define RWSEM_ANONYMOUSLY_OWNED (1UL << 1)
  28
  29 #ifdef CONFIG_DEBUG_RWSEMS
  30 # define DEBUG_RWSEMS_WARN_ON(c)        DEBUG_LOCKS_WARN_ON(c)
  31 #else
  32 # define DEBUG_RWSEMS_WARN_ON(c)
  33 #endif
  34
  35 /*
  36  * R/W semaphores originally for PPC using the stuff in lib/rwsem.c.
  37  * Adapted largely from include/asm-i386/rwsem.h
  38  * by Paul Mackerras <paulus@samba.org>.
  39  */
  40
  41 /*
  42  * the semaphore definition
  43  */
  44 #ifdef CONFIG_64BIT
  45 # define RWSEM_ACTIVE_MASK              0xffffffffL
  46 #else
  47 # define RWSEM_ACTIVE_MASK              0x0000ffffL
  48 #endif
  49
  50 #define RWSEM_ACTIVE_BIAS               0x00000001L
  51 #define RWSEM_WAITING_BIAS              (-RWSEM_ACTIVE_MASK-1)
  52 #define RWSEM_ACTIVE_READ_BIAS          RWSEM_ACTIVE_BIAS
  53 #define RWSEM_ACTIVE_WRITE_BIAS         (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
  54
  55 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
  56 /*
  57  * All writes to owner are protected by WRITE_ONCE() to make sure that
  58  * store tearing can't happen as optimistic spinners may read and use
  59  * the owner value concurrently without lock. Read from owner, however,
  60  * may not need READ_ONCE() as long as the pointer value is only used
  61  * for comparison and isn't being dereferenced.
  62  */
  63 static inline void rwsem_set_owner(struct rw_semaphore *sem)
  64 {
  65         WRITE_ONCE(sem->owner, current);
  66 }
  67
  68 static inline void rwsem_clear_owner(struct rw_semaphore *sem)
  69 {
  70         WRITE_ONCE(sem->owner, NULL);
  71 }
  72
  73 /*
  74  * The task_struct pointer of the last owning reader will be left in
  75  * the owner field.
  76  *
  77  * Note that the owner value just indicates the task has owned the rwsem
  78  * previously, it may not be the real owner or one of the real owners
  79  * anymore when that field is examined, so take it with a grain of salt.
  80  */
  81 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
  82                                             struct task_struct *owner)
  83 {
  84         unsigned long val = (unsigned long)owner | RWSEM_READER_OWNED
  85                                                  | RWSEM_ANONYMOUSLY_OWNED;
  86
  87         WRITE_ONCE(sem->owner, (struct task_struct *)val);
  88 }
  89
  90 static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
  91 {
  92         __rwsem_set_reader_owned(sem, current);
  93 }
  94
  95 /*
  96  * Return true if the a rwsem waiter can spin on the rwsem's owner
  97  * and steal the lock, i.e. the lock is not anonymously owned.
  98  * N.B. !owner is considered spinnable.
  99  */
 100 static inline bool is_rwsem_owner_spinnable(struct task_struct *owner)
 101 {
 102         return !((unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED);
 103 }
 104
 105 /*
 106  * Return true if rwsem is owned by an anonymous writer or readers.
 107  */
 108 static inline bool rwsem_has_anonymous_owner(struct task_struct *owner)
 109 {
 110         return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED;
 111 }
 112
 113 #ifdef CONFIG_DEBUG_RWSEMS
 114 /*
 115  * With CONFIG_DEBUG_RWSEMS configured, it will make sure that if there
 116  * is a task pointer in owner of a reader-owned rwsem, it will be the
 117  * real owner or one of the real owners. The only exception is when the
 118  * unlock is done by up_read_non_owner().
 119  */
 120 #define rwsem_clear_reader_owned rwsem_clear_reader_owned
 121 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)
 122 {
 123         unsigned long val = (unsigned long)current | RWSEM_READER_OWNED
 124                                                    | RWSEM_ANONYMOUSLY_OWNED;
 125         if (READ_ONCE(sem->owner) == (struct task_struct *)val)
 126                 cmpxchg_relaxed((unsigned long *)&sem->owner, val,
 127                                 RWSEM_READER_OWNED | RWSEM_ANONYMOUSLY_OWNED);
 128 }
 129 #endif
 130
 131 #else
 132 static inline void rwsem_set_owner(struct rw_semaphore *sem)
 133 {
 134 }
 135
 136 static inline void rwsem_clear_owner(struct rw_semaphore *sem)
 137 {
 138 }
 139
 140 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
 141                                            struct task_struct *owner)
 142 {
 143 }
 144
 145 static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
 146 {
 147 }
 148 #endif
 149
 150 #ifndef rwsem_clear_reader_owned
 151 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)
 152 {
 153 }
 154 #endif
 155
 156 /*
 157  * lock for reading
 158  */
 159 static inline void __down_read(struct rw_semaphore *sem)
 160 {
 161         if (unlikely(atomic_long_inc_return_acquire(&sem->count) <= 0))
 162                 rwsem_down_read_failed(sem);
 163         else
 164                 rwsem_set_reader_owned(sem);
 165 }
 166
 167 static inline int __down_read_killable(struct rw_semaphore *sem)
 168 {
 169         if (unlikely(atomic_long_inc_return_acquire(&sem->count) <= 0)) {
 170                 if (IS_ERR(rwsem_down_read_failed_killable(sem)))
 171                         return -EINTR;
 172         } else {
 173                 rwsem_set_reader_owned(sem);
 174         }
 175         return 0;
 176 }
 177
 178 static inline int __down_read_trylock(struct rw_semaphore *sem)
 179 {
 180         /*
 181          * Optimize for the case when the rwsem is not locked at all.
 182          */
 183         long tmp = RWSEM_UNLOCKED_VALUE;
 184
 185         do {
 186                 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp,
 187                                         tmp + RWSEM_ACTIVE_READ_BIAS)) {
 188                         rwsem_set_reader_owned(sem);
 189                         return 1;
 190                 }
 191         } while (tmp >= 0);
 192         return 0;
 193 }
 194
 195 /*
 196  * lock for writing
 197  */
 198 static inline void __down_write(struct rw_semaphore *sem)
 199 {
 200         long tmp;
 201
 202         tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS,
 203                                              &sem->count);
 204         if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
 205                 rwsem_down_write_failed(sem);
 206         rwsem_set_owner(sem);
 207 }
 208
 209 static inline int __down_write_killable(struct rw_semaphore *sem)
 210 {
 211         long tmp;
 212
 213         tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS,
 214                                              &sem->count);
 215         if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
 216                 if (IS_ERR(rwsem_down_write_failed_killable(sem)))
 217                         return -EINTR;
 218         rwsem_set_owner(sem);
 219         return 0;
 220 }
 221
 222 static inline int __down_write_trylock(struct rw_semaphore *sem)
 223 {
 224         long tmp;
 225
 226         tmp = atomic_long_cmpxchg_acquire(&sem->count, RWSEM_UNLOCKED_VALUE,
 227                       RWSEM_ACTIVE_WRITE_BIAS);
 228         if (tmp == RWSEM_UNLOCKED_VALUE) {
 229                 rwsem_set_owner(sem);
 230                 return true;
 231         }
 232         return false;
 233 }
 234
 235 /*
 236  * unlock after reading
 237  */
 238 static inline void __up_read(struct rw_semaphore *sem)
 239 {
 240         long tmp;
 241
 242         rwsem_clear_reader_owned(sem);
 243         tmp = atomic_long_dec_return_release(&sem->count);
 244         if (unlikely(tmp < -1 && (tmp & RWSEM_ACTIVE_MASK) == 0))
 245                 rwsem_wake(sem);
 246 }
 247
 248 /*
 249  * unlock after writing
 250  */
 251 static inline void __up_write(struct rw_semaphore *sem)
 252 {
 253         rwsem_clear_owner(sem);
 254         if (unlikely(atomic_long_sub_return_release(RWSEM_ACTIVE_WRITE_BIAS,
 255                                                     &sem->count) < 0))
 256                 rwsem_wake(sem);
 257 }
 258
 259 /*
 260  * downgrade write lock to read lock
 261  */
 262 static inline void __downgrade_write(struct rw_semaphore *sem)
 263 {
 264         long tmp;
 265
 266         /*
 267          * When downgrading from exclusive to shared ownership,
 268          * anything inside the write-locked region cannot leak
 269          * into the read side. In contrast, anything in the
 270          * read-locked region is ok to be re-ordered into the
 271          * write side. As such, rely on RELEASE semantics.
 272          */
 273         tmp = atomic_long_add_return_release(-RWSEM_WAITING_BIAS, &sem->count);
 274         rwsem_set_reader_owned(sem);
 275         if (tmp < 0)
 276                 rwsem_downgrade_wake(sem);
 277 }