rt-patches/0361-kernel-futex-don-t-deboost-too-early.patch

   1 From e8bb9507c2cc702c944c185647fa110bba74f73f Mon Sep 17 00:00:00 2001
   2 From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   3 Date: Thu, 29 Sep 2016 18:49:22 +0200
   4 Subject: [PATCH 361/366] kernel/futex: don't deboost too early
   5
   6 The sequence:
   7  T1 holds futex
   8  T2 blocks on futex and boosts T1
   9  T1 unlocks futex and holds hb->lock
  10  T1 unlocks rt mutex, so T1 has no more pi waiters
  11  T3 blocks on hb->lock and adds itself to the pi waiters list of T1
  12  T1 unlocks hb->lock and deboosts itself
  13  T4 preempts T1 so the wakeup of T2 gets delayed
  14
  15 As a workaround I attempt here do unlock the hb->lock without a deboost
  16 and perform the deboost after the wake up of the waiter.
  17
  18 Cc: stable-rt@vger.kernel.org
  19 Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
  20 Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
  21 ---
  22  include/linux/spinlock.h    |  6 +++++
  23  include/linux/spinlock_rt.h |  2 ++
  24  kernel/futex.c              |  2 +-
  25  kernel/locking/rtmutex.c    | 53 +++++++++++++++++++++++++++++++++++++++------
  26  4 files changed, 55 insertions(+), 8 deletions(-)
  27
  28 diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
  29 index b241cc0..02928fa 100644
  30 --- a/include/linux/spinlock.h
  31 +++ b/include/linux/spinlock.h
  32 @@ -355,6 +355,12 @@ static __always_inline void spin_unlock(spinlock_t *lock)
  33         raw_spin_unlock(&lock->rlock);
  34  }
  35
  36 +static __always_inline int spin_unlock_no_deboost(spinlock_t *lock)
  37 +{
  38 +       raw_spin_unlock(&lock->rlock);
  39 +       return 0;
  40 +}
  41 +
  42  static __always_inline void spin_unlock_bh(spinlock_t *lock)
  43  {
  44         raw_spin_unlock_bh(&lock->rlock);
  45 diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h
  46 index 3b28255..7eb8758 100644
  47 --- a/include/linux/spinlock_rt.h
  48 +++ b/include/linux/spinlock_rt.h
  49 @@ -26,6 +26,7 @@ extern void __lockfunc rt_spin_lock(spinlock_t *lock);
  50  extern unsigned long __lockfunc rt_spin_lock_trace_flags(spinlock_t *lock);
  51  extern void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass);
  52  extern void __lockfunc rt_spin_unlock(spinlock_t *lock);
  53 +extern int __lockfunc rt_spin_unlock_no_deboost(spinlock_t *lock);
  54  extern void __lockfunc rt_spin_unlock_wait(spinlock_t *lock);
  55  extern int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags);
  56  extern int __lockfunc rt_spin_trylock_bh(spinlock_t *lock);
  57 @@ -112,6 +113,7 @@ static inline unsigned long spin_lock_trace_flags(spinlock_t *lock)
  58  #define spin_lock_nest_lock(lock, nest_lock) spin_lock_nested(lock, 0)
  59
  60  #define spin_unlock(lock)                      rt_spin_unlock(lock)
  61 +#define spin_unlock_no_deboost(lock)           rt_spin_unlock_no_deboost(lock)
  62
  63  #define spin_unlock_bh(lock)                           \
  64         do {                                            \
  65 diff --git a/kernel/futex.c b/kernel/futex.c
  66 index ad38af0..0596234 100644
  67 --- a/kernel/futex.c
  68 +++ b/kernel/futex.c
  69 @@ -1288,7 +1288,7 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this,
  70          * deboost first (and lose our higher priority), then the task might get
  71          * scheduled away before the wake up can take place.
  72          */
  73 -       spin_unlock(&hb->lock);
  74 +       deboost |= spin_unlock_no_deboost(&hb->lock);
  75         wake_up_q(&wake_q);
  76         wake_up_q_sleeper(&wake_sleeper_q);
  77         if (deboost)
  78 diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
  79 index fde5e54..6759a79 100644
  80 --- a/kernel/locking/rtmutex.c
  81 +++ b/kernel/locking/rtmutex.c
  82 @@ -939,13 +939,14 @@ static inline void rt_spin_lock_fastlock(struct rt_mutex *lock,
  83                 slowfn(lock, do_mig_dis);
  84  }
  85
  86 -static inline void rt_spin_lock_fastunlock(struct rt_mutex *lock,
  87 -                                          void  (*slowfn)(struct rt_mutex *lock))
  88 +static inline int rt_spin_lock_fastunlock(struct rt_mutex *lock,
  89 +                                          int  (*slowfn)(struct rt_mutex *lock))
  90  {
  91 -       if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
  92 +       if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) {
  93                 rt_mutex_deadlock_account_unlock(current);
  94 -       else
  95 -               slowfn(lock);
  96 +               return 0;
  97 +       }
  98 +       return slowfn(lock);
  99  }
 100  #ifdef CONFIG_SMP
 101  /*
 102 @@ -1086,7 +1087,7 @@ static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
 103  /*
 104   * Slow path to release a rt_mutex spin_lock style
 105   */
 106 -static void  noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock)
 107 +static int noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock)
 108  {
 109         unsigned long flags;
 110         WAKE_Q(wake_q);
 111 @@ -1101,7 +1102,7 @@ static void  noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock)
 112         if (!rt_mutex_has_waiters(lock)) {
 113                 lock->owner = NULL;
 114                 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
 115 -               return;
 116 +               return 0;
 117         }
 118
 119         mark_wakeup_next_waiter(&wake_q, &wake_sleeper_q, lock);
 120 @@ -1112,6 +1113,33 @@ static void  noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock)
 121
 122         /* Undo pi boosting.when necessary */
 123         rt_mutex_adjust_prio(current);
 124 +       return 0;
 125 +}
 126 +
 127 +static int noinline __sched rt_spin_lock_slowunlock_no_deboost(struct rt_mutex *lock)
 128 +{
 129 +       unsigned long flags;
 130 +       WAKE_Q(wake_q);
 131 +       WAKE_Q(wake_sleeper_q);
 132 +
 133 +       raw_spin_lock_irqsave(&lock->wait_lock, flags);
 134 +
 135 +       debug_rt_mutex_unlock(lock);
 136 +
 137 +       rt_mutex_deadlock_account_unlock(current);
 138 +
 139 +       if (!rt_mutex_has_waiters(lock)) {
 140 +               lock->owner = NULL;
 141 +               raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
 142 +               return 0;
 143 +       }
 144 +
 145 +       mark_wakeup_next_waiter(&wake_q, &wake_sleeper_q, lock);
 146 +
 147 +       raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
 148 +       wake_up_q(&wake_q);
 149 +       wake_up_q_sleeper(&wake_sleeper_q);
 150 +       return 1;
 151  }
 152
 153  void __lockfunc rt_spin_lock__no_mg(spinlock_t *lock)
 154 @@ -1166,6 +1194,17 @@ void __lockfunc rt_spin_unlock(spinlock_t *lock)
 155  }
 156  EXPORT_SYMBOL(rt_spin_unlock);
 157
 158 +int __lockfunc rt_spin_unlock_no_deboost(spinlock_t *lock)
 159 +{
 160 +       int ret;
 161 +
 162 +       /* NOTE: we always pass in '1' for nested, for simplicity */
 163 +       spin_release(&lock->dep_map, 1, _RET_IP_);
 164 +       ret = rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock_no_deboost);
 165 +       migrate_enable();
 166 +       return ret;
 167 +}
 168 +
 169  void __lockfunc __rt_spin_unlock(struct rt_mutex *lock)
 170  {
 171         rt_spin_lock_fastunlock(lock, rt_spin_lock_slowunlock);
 172 --
 173 1.9.1
 174