kernel/locking/rwsem.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * The least significant 2 bits of the owner value has the following
   4  * meanings when set.
   5  *  - RWSEM_READER_OWNED (bit 0): The rwsem is owned by readers
   6  *  - RWSEM_ANONYMOUSLY_OWNED (bit 1): The rwsem is anonymously owned,
   7  *    i.e. the owner(s) cannot be readily determined. It can be reader
   8  *    owned or the owning writer is indeterminate.
   9  *
  10  * When a writer acquires a rwsem, it puts its task_struct pointer
  11  * into the owner field. It is cleared after an unlock.
  12  *
  13  * When a reader acquires a rwsem, it will also puts its task_struct
  14  * pointer into the owner field with both the RWSEM_READER_OWNED and
  15  * RWSEM_ANONYMOUSLY_OWNED bits set. On unlock, the owner field will
  16  * largely be left untouched. So for a free or reader-owned rwsem,
  17  * the owner value may contain information about the last reader that
  18  * acquires the rwsem. The anonymous bit is set because that particular
  19  * reader may or may not still own the lock.
  20  *
  21  * That information may be helpful in debugging cases where the system
  22  * seems to hang on a reader owned rwsem especially if only one reader
  23  * is involved. Ideally we would like to track all the readers that own
  24  * a rwsem, but the overhead is simply too big.
  25  */
  26 #define RWSEM_READER_OWNED      (1UL << 0)
  27 #define RWSEM_ANONYMOUSLY_OWNED (1UL << 1)
  28
  29 #ifdef CONFIG_DEBUG_RWSEMS
  30 # define DEBUG_RWSEMS_WARN_ON(c)        DEBUG_LOCKS_WARN_ON(c)
  31 #else
  32 # define DEBUG_RWSEMS_WARN_ON(c)
  33 #endif
  34
  35 /*
  36  * R/W semaphores originally for PPC using the stuff in lib/rwsem.c.
  37  * Adapted largely from include/asm-i386/rwsem.h
  38  * by Paul Mackerras <paulus@samba.org>.
  39  */
  40
  41 /*
  42  * the semaphore definition
  43  */
  44 #ifdef CONFIG_64BIT
  45 # define RWSEM_ACTIVE_MASK              0xffffffffL
  46 #else
  47 # define RWSEM_ACTIVE_MASK              0x0000ffffL
  48 #endif
  49
  50 #define RWSEM_ACTIVE_BIAS               0x00000001L
  51 #define RWSEM_WAITING_BIAS              (-RWSEM_ACTIVE_MASK-1)
  52 #define RWSEM_ACTIVE_READ_BIAS          RWSEM_ACTIVE_BIAS
  53 #define RWSEM_ACTIVE_WRITE_BIAS         (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
  54
  55 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
  56 /*
  57  * All writes to owner are protected by WRITE_ONCE() to make sure that
  58  * store tearing can't happen as optimistic spinners may read and use
  59  * the owner value concurrently without lock. Read from owner, however,
  60  * may not need READ_ONCE() as long as the pointer value is only used
  61  * for comparison and isn't being dereferenced.
  62  */
  63 static inline void rwsem_set_owner(struct rw_semaphore *sem)
  64 {
  65         WRITE_ONCE(sem->owner, current);
  66 }
  67
  68 static inline void rwsem_clear_owner(struct rw_semaphore *sem)
  69 {
  70         WRITE_ONCE(sem->owner, NULL);
  71 }
  72
  73 /*
  74  * The task_struct pointer of the last owning reader will be left in
  75  * the owner field.
  76  *
  77  * Note that the owner value just indicates the task has owned the rwsem
  78  * previously, it may not be the real owner or one of the real owners
  79  * anymore when that field is examined, so take it with a grain of salt.
  80  */
  81 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
  82                                             struct task_struct *owner)
  83 {
  84         unsigned long val = (unsigned long)owner | RWSEM_READER_OWNED
  85                                                  | RWSEM_ANONYMOUSLY_OWNED;
  86
  87         WRITE_ONCE(sem->owner, (struct task_struct *)val);
  88 }
  89
  90 static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
  91 {
  92         __rwsem_set_reader_owned(sem, current);
  93 }
  94
  95 /*
  96  * Return true if the a rwsem waiter can spin on the rwsem's owner
  97  * and steal the lock, i.e. the lock is not anonymously owned.
  98  * N.B. !owner is considered spinnable.
  99  */
 100 static inline bool is_rwsem_owner_spinnable(struct task_struct *owner)
 101 {
 102         return !((unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED);
 103 }
 104
 105 /*
 106  * Return true if rwsem is owned by an anonymous writer or readers.
 107  */
 108 static inline bool rwsem_has_anonymous_owner(struct task_struct *owner)
 109 {
 110         return (unsigned long)owner & RWSEM_ANONYMOUSLY_OWNED;
 111 }
 112
 113 #ifdef CONFIG_DEBUG_RWSEMS
 114 /*
 115  * With CONFIG_DEBUG_RWSEMS configured, it will make sure that if there
 116  * is a task pointer in owner of a reader-owned rwsem, it will be the
 117  * real owner or one of the real owners. The only exception is when the
 118  * unlock is done by up_read_non_owner().
 119  */
 120 #define rwsem_clear_reader_owned rwsem_clear_reader_owned
 121 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)
 122 {
 123         unsigned long val = (unsigned long)current | RWSEM_READER_OWNED
 124                                                    | RWSEM_ANONYMOUSLY_OWNED;
 125         if (READ_ONCE(sem->owner) == (struct task_struct *)val)
 126                 cmpxchg_relaxed((unsigned long *)&sem->owner, val,
 127                                 RWSEM_READER_OWNED | RWSEM_ANONYMOUSLY_OWNED);
 128 }
 129 #endif
 130
 131 #else
 132 static inline void rwsem_set_owner(struct rw_semaphore *sem)
 133 {
 134 }
 135
 136 static inline void rwsem_clear_owner(struct rw_semaphore *sem)
 137 {
 138 }
 139
 140 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem,
 141                                            struct task_struct *owner)
 142 {
 143 }
 144
 145 static inline void rwsem_set_reader_owned(struct rw_semaphore *sem)
 146 {
 147 }
 148 #endif
 149
 150 #ifndef rwsem_clear_reader_owned
 151 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem)
 152 {
 153 }
 154 #endif
 155
 156 #ifdef CONFIG_RWSEM_XCHGADD_ALGORITHM
 157 /*
 158  * lock for reading
 159  */
 160 static inline void __down_read(struct rw_semaphore *sem)
 161 {
 162         if (unlikely(atomic_long_inc_return_acquire(&sem->count) <= 0))
 163                 rwsem_down_read_failed(sem);
 164 }
 165
 166 static inline int __down_read_killable(struct rw_semaphore *sem)
 167 {
 168         if (unlikely(atomic_long_inc_return_acquire(&sem->count) <= 0)) {
 169                 if (IS_ERR(rwsem_down_read_failed_killable(sem)))
 170                         return -EINTR;
 171         }
 172
 173         return 0;
 174 }
 175
 176 static inline int __down_read_trylock(struct rw_semaphore *sem)
 177 {
 178         long tmp;
 179
 180         while ((tmp = atomic_long_read(&sem->count)) >= 0) {
 181                 if (tmp == atomic_long_cmpxchg_acquire(&sem->count, tmp,
 182                                    tmp + RWSEM_ACTIVE_READ_BIAS)) {
 183                         return 1;
 184                 }
 185         }
 186         return 0;
 187 }
 188
 189 /*
 190  * lock for writing
 191  */
 192 static inline void __down_write(struct rw_semaphore *sem)
 193 {
 194         long tmp;
 195
 196         tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS,
 197                                              &sem->count);
 198         if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
 199                 rwsem_down_write_failed(sem);
 200 }
 201
 202 static inline int __down_write_killable(struct rw_semaphore *sem)
 203 {
 204         long tmp;
 205
 206         tmp = atomic_long_add_return_acquire(RWSEM_ACTIVE_WRITE_BIAS,
 207                                              &sem->count);
 208         if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
 209                 if (IS_ERR(rwsem_down_write_failed_killable(sem)))
 210                         return -EINTR;
 211         return 0;
 212 }
 213
 214 static inline int __down_write_trylock(struct rw_semaphore *sem)
 215 {
 216         long tmp;
 217
 218         tmp = atomic_long_cmpxchg_acquire(&sem->count, RWSEM_UNLOCKED_VALUE,
 219                       RWSEM_ACTIVE_WRITE_BIAS);
 220         return tmp == RWSEM_UNLOCKED_VALUE;
 221 }
 222
 223 /*
 224  * unlock after reading
 225  */
 226 static inline void __up_read(struct rw_semaphore *sem)
 227 {
 228         long tmp;
 229
 230         tmp = atomic_long_dec_return_release(&sem->count);
 231         if (unlikely(tmp < -1 && (tmp & RWSEM_ACTIVE_MASK) == 0))
 232                 rwsem_wake(sem);
 233 }
 234
 235 /*
 236  * unlock after writing
 237  */
 238 static inline void __up_write(struct rw_semaphore *sem)
 239 {
 240         if (unlikely(atomic_long_sub_return_release(RWSEM_ACTIVE_WRITE_BIAS,
 241                                                     &sem->count) < 0))
 242                 rwsem_wake(sem);
 243 }
 244
 245 /*
 246  * downgrade write lock to read lock
 247  */
 248 static inline void __downgrade_write(struct rw_semaphore *sem)
 249 {
 250         long tmp;
 251
 252         /*
 253          * When downgrading from exclusive to shared ownership,
 254          * anything inside the write-locked region cannot leak
 255          * into the read side. In contrast, anything in the
 256          * read-locked region is ok to be re-ordered into the
 257          * write side. As such, rely on RELEASE semantics.
 258          */
 259         tmp = atomic_long_add_return_release(-RWSEM_WAITING_BIAS, &sem->count);
 260         if (tmp < 0)
 261                 rwsem_downgrade_wake(sem);
 262 }
 263
 264 #endif /* CONFIG_RWSEM_XCHGADD_ALGORITHM */