sched/wake_q: Reduce reference counting for special users

[linux.git] / kernel / futex.c

diff --git a/kernel/futex.c b/kernel/futex.c
index 054105854e0e38fbab1fbaee8fc743729526de2c..2abe1a0b30626c6161e8ec18e57d5ba6b04a74b5 100644 (file)
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -321,12 +321,8 @@ static int __init fail_futex_debugfs(void)
        if (IS_ERR(dir))
                return PTR_ERR(dir);
 
-       if (!debugfs_create_bool("ignore-private", mode, dir,
-                                &fail_futex.ignore_private)) {
-               debugfs_remove_recursive(dir);
-               return -ENOMEM;
-       }
-
+       debugfs_create_bool("ignore-private", mode, dir,
+                           &fail_futex.ignore_private);
        return 0;
 }
 
@@ -481,13 +477,18 @@ static void drop_futex_key_refs(union futex_key *key)
        }
 }
 
+enum futex_access {
+       FUTEX_READ,
+       FUTEX_WRITE
+};
+
 /**
  * get_futex_key() - Get parameters which are the keys for a futex
  * @uaddr:     virtual address of the futex
  * @fshared:   0 for a PROCESS_PRIVATE futex, 1 for PROCESS_SHARED
  * @key:       address where result is stored.
- * @rw:                mapping needs to be read/write (values: VERIFY_READ,
- *              VERIFY_WRITE)
+ * @rw:                mapping needs to be read/write (values: FUTEX_READ,
+ *              FUTEX_WRITE)
  *
  * Return: a negative error code or 0
  *
@@ -500,7 +501,7 @@ static void drop_futex_key_refs(union futex_key *key)
  * lock_page() might sleep, the caller should not hold a spinlock.
  */
 static int
-get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw)
+get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, enum futex_access rw)
 {
        unsigned long address = (unsigned long)uaddr;
        struct mm_struct *mm = current->mm;
@@ -516,7 +517,7 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw)
                return -EINVAL;
        address -= key->both.offset;
 
-       if (unlikely(!access_ok(rw, uaddr, sizeof(u32))))
+       if (unlikely(!access_ok(uaddr, sizeof(u32))))
                return -EFAULT;
 
        if (unlikely(should_fail_futex(fshared)))
@@ -546,7 +547,7 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw)
         * If write access is not required (eg. FUTEX_WAIT), try
         * and get read-only access.
         */
-       if (err == -EFAULT && rw == VERIFY_READ) {
+       if (err == -EFAULT && rw == FUTEX_READ) {
                err = get_user_pages_fast(address, 1, 0, &page);
                ro = 1;
        }
@@ -1447,11 +1448,7 @@ static void mark_wake_futex(struct wake_q_head *wake_q, struct futex_q *q)
        if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))
                return;
 
-       /*
-        * Queue the task for later wakeup for after we've released
-        * the hb->lock. wake_q_add() grabs reference to p.
-        */
-       wake_q_add(wake_q, p);
+       get_task_struct(p);
        __unqueue_futex(q);
        /*
         * The waiting task can free the futex_q as soon as q->lock_ptr = NULL
@@ -1461,6 +1458,12 @@ static void mark_wake_futex(struct wake_q_head *wake_q, struct futex_q *q)
         * plist_del in __unqueue_futex().
         */
        smp_store_release(&q->lock_ptr, NULL);
+
+       /*
+        * Queue the task for later wakeup for after we've released
+        * the hb->lock. wake_q_add() grabs reference to p.
+        */
+       wake_q_add_safe(wake_q, p);
 }
 
 /*
@@ -1583,7 +1586,7 @@ futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
        if (!bitset)
                return -EINVAL;
 
-       ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, VERIFY_READ);
+       ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_READ);
        if (unlikely(ret != 0))
                goto out;
 
@@ -1642,7 +1645,7 @@ static int futex_atomic_op_inuser(unsigned int encoded_op, u32 __user *uaddr)
                oparg = 1 << oparg;
        }
 
-       if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
+       if (!access_ok(uaddr, sizeof(u32)))
                return -EFAULT;
 
        ret = arch_futex_atomic_op_inuser(op, oparg, &oldval, uaddr);
@@ -1682,10 +1685,10 @@ futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2,
        DEFINE_WAKE_Q(wake_q);
 
 retry:
-       ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, VERIFY_READ);
+       ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);
        if (unlikely(ret != 0))
                goto out;
-       ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE);
+       ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);
        if (unlikely(ret != 0))
                goto out_put_key1;
 
@@ -1961,11 +1964,11 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags,
        }
 
 retry:
-       ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, VERIFY_READ);
+       ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);
        if (unlikely(ret != 0))
                goto out;
        ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2,
-                           requeue_pi ? VERIFY_WRITE : VERIFY_READ);
+                           requeue_pi ? FUTEX_WRITE : FUTEX_READ);
        if (unlikely(ret != 0))
                goto out_put_key1;
 
@@ -2634,7 +2637,7 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
         * while the syscall executes.
         */
 retry:
-       ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, VERIFY_READ);
+       ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ);
        if (unlikely(ret != 0))
                return ret;
 
@@ -2793,7 +2796,7 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags,
        }
 
 retry:
-       ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, VERIFY_WRITE);
+       ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, FUTEX_WRITE);
        if (unlikely(ret != 0))
                goto out;
 
@@ -2972,7 +2975,7 @@ static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
        if ((uval & FUTEX_TID_MASK) != vpid)
                return -EPERM;
 
-       ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, VERIFY_WRITE);
+       ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_WRITE);
        if (ret)
                return ret;
 
@@ -3199,7 +3202,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,
         */
        rt_mutex_init_waiter(&rt_waiter);
 
-       ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE);
+       ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);
        if (unlikely(ret != 0))
                goto out;