rt-patches/0140-wait-simple-Simple-waitqueue-implementation.patch

   1 From 5dd7edbd6980c6139624515d75bc93f89dff142a Mon Sep 17 00:00:00 2001
   2 From: Thomas Gleixner <tglx@linutronix.de>
   3 Date: Mon, 12 Dec 2011 12:29:04 +0100
   4 Subject: [PATCH 140/366] wait-simple: Simple waitqueue implementation
   5
   6 wait_queue is a swiss army knife and in most of the cases the
   7 complexity is not needed. For RT waitqueues are a constant source of
   8 trouble as we can't convert the head lock to a raw spinlock due to
   9 fancy and long lasting callbacks.
  10
  11 Provide a slim version, which allows RT to replace wait queues. This
  12 should go mainline as well, as it lowers memory consumption and
  13 runtime overhead.
  14
  15 Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
  16
  17 smp_mb() added by Steven Rostedt to fix a race condition with swait
  18 wakeups vs adding items to the list.
  19 ---
  20  include/linux/wait-simple.h | 207 ++++++++++++++++++++++++++++++++++++++++++++
  21  kernel/sched/Makefile       |   2 +-
  22  kernel/sched/wait-simple.c  | 115 ++++++++++++++++++++++++
  23  3 files changed, 323 insertions(+), 1 deletion(-)
  24  create mode 100644 include/linux/wait-simple.h
  25  create mode 100644 kernel/sched/wait-simple.c
  26
  27 diff --git a/include/linux/wait-simple.h b/include/linux/wait-simple.h
  28 new file mode 100644
  29 index 0000000..f86bca2
  30 --- /dev/null
  31 +++ b/include/linux/wait-simple.h
  32 @@ -0,0 +1,207 @@
  33 +#ifndef _LINUX_WAIT_SIMPLE_H
  34 +#define _LINUX_WAIT_SIMPLE_H
  35 +
  36 +#include <linux/spinlock.h>
  37 +#include <linux/list.h>
  38 +
  39 +#include <asm/current.h>
  40 +
  41 +struct swaiter {
  42 +       struct task_struct      *task;
  43 +       struct list_head        node;
  44 +};
  45 +
  46 +#define DEFINE_SWAITER(name)                                   \
  47 +       struct swaiter name = {                                 \
  48 +               .task   = current,                              \
  49 +               .node   = LIST_HEAD_INIT((name).node),          \
  50 +       }
  51 +
  52 +struct swait_head {
  53 +       raw_spinlock_t          lock;
  54 +       struct list_head        list;
  55 +};
  56 +
  57 +#define SWAIT_HEAD_INITIALIZER(name) {                         \
  58 +               .lock   = __RAW_SPIN_LOCK_UNLOCKED(name.lock),  \
  59 +               .list   = LIST_HEAD_INIT((name).list),          \
  60 +       }
  61 +
  62 +#define DEFINE_SWAIT_HEAD(name)                                        \
  63 +       struct swait_head name = SWAIT_HEAD_INITIALIZER(name)
  64 +
  65 +extern void __init_swait_head(struct swait_head *h, struct lock_class_key *key);
  66 +
  67 +#define init_swait_head(swh)                                   \
  68 +       do {                                                    \
  69 +               static struct lock_class_key __key;             \
  70 +                                                               \
  71 +               __init_swait_head((swh), &__key);               \
  72 +       } while (0)
  73 +
  74 +/*
  75 + * Waiter functions
  76 + */
  77 +extern void swait_prepare_locked(struct swait_head *head, struct swaiter *w);
  78 +extern void swait_prepare(struct swait_head *head, struct swaiter *w, int state);
  79 +extern void swait_finish_locked(struct swait_head *head, struct swaiter *w);
  80 +extern void swait_finish(struct swait_head *head, struct swaiter *w);
  81 +
  82 +/* Check whether a head has waiters enqueued */
  83 +static inline bool swaitqueue_active(struct swait_head *h)
  84 +{
  85 +       /* Make sure the condition is visible before checking list_empty() */
  86 +       smp_mb();
  87 +       return !list_empty(&h->list);
  88 +}
  89 +
  90 +/*
  91 + * Wakeup functions
  92 + */
  93 +extern unsigned int __swait_wake(struct swait_head *head, unsigned int state, unsigned int num);
  94 +extern unsigned int __swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num);
  95 +
  96 +#define swait_wake(head)                       __swait_wake(head, TASK_NORMAL, 1)
  97 +#define swait_wake_interruptible(head)         __swait_wake(head, TASK_INTERRUPTIBLE, 1)
  98 +#define swait_wake_all(head)                   __swait_wake(head, TASK_NORMAL, 0)
  99 +#define swait_wake_all_interruptible(head)     __swait_wake(head, TASK_INTERRUPTIBLE, 0)
 100 +
 101 +/*
 102 + * Event API
 103 + */
 104 +#define __swait_event(wq, condition)                                   \
 105 +do {                                                                   \
 106 +       DEFINE_SWAITER(__wait);                                         \
 107 +                                                                       \
 108 +       for (;;) {                                                      \
 109 +               swait_prepare(&wq, &__wait, TASK_UNINTERRUPTIBLE);      \
 110 +               if (condition)                                          \
 111 +                       break;                                          \
 112 +               schedule();                                             \
 113 +       }                                                               \
 114 +       swait_finish(&wq, &__wait);                                     \
 115 +} while (0)
 116 +
 117 +/**
 118 + * swait_event - sleep until a condition gets true
 119 + * @wq: the waitqueue to wait on
 120 + * @condition: a C expression for the event to wait for
 121 + *
 122 + * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 123 + * @condition evaluates to true. The @condition is checked each time
 124 + * the waitqueue @wq is woken up.
 125 + *
 126 + * wake_up() has to be called after changing any variable that could
 127 + * change the result of the wait condition.
 128 + */
 129 +#define swait_event(wq, condition)                                     \
 130 +do {                                                                   \
 131 +       if (condition)                                                  \
 132 +               break;                                                  \
 133 +       __swait_event(wq, condition);                                   \
 134 +} while (0)
 135 +
 136 +#define __swait_event_interruptible(wq, condition, ret)                        \
 137 +do {                                                                   \
 138 +       DEFINE_SWAITER(__wait);                                         \
 139 +                                                                       \
 140 +       for (;;) {                                                      \
 141 +               swait_prepare(&wq, &__wait, TASK_INTERRUPTIBLE);        \
 142 +               if (condition)                                          \
 143 +                       break;                                          \
 144 +               if (signal_pending(current)) {                          \
 145 +                       ret = -ERESTARTSYS;                             \
 146 +                       break;                                          \
 147 +               }                                                       \
 148 +               schedule();                                             \
 149 +       }                                                               \
 150 +       swait_finish(&wq, &__wait);                                     \
 151 +} while (0)
 152 +
 153 +#define __swait_event_interruptible_timeout(wq, condition, ret)                \
 154 +do {                                                                   \
 155 +       DEFINE_SWAITER(__wait);                                         \
 156 +                                                                       \
 157 +       for (;;) {                                                      \
 158 +               swait_prepare(&wq, &__wait, TASK_INTERRUPTIBLE);        \
 159 +               if (condition)                                          \
 160 +                       break;                                          \
 161 +               if (signal_pending(current)) {                          \
 162 +                       ret = -ERESTARTSYS;                             \
 163 +                       break;                                          \
 164 +               }                                                       \
 165 +               ret = schedule_timeout(ret);                            \
 166 +               if (!ret)                                               \
 167 +                       break;                                          \
 168 +       }                                                               \
 169 +       swait_finish(&wq, &__wait);                                     \
 170 +} while (0)
 171 +
 172 +/**
 173 + * swait_event_interruptible - sleep until a condition gets true
 174 + * @wq: the waitqueue to wait on
 175 + * @condition: a C expression for the event to wait for
 176 + *
 177 + * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 178 + * @condition evaluates to true. The @condition is checked each time
 179 + * the waitqueue @wq is woken up.
 180 + *
 181 + * wake_up() has to be called after changing any variable that could
 182 + * change the result of the wait condition.
 183 + */
 184 +#define swait_event_interruptible(wq, condition)                       \
 185 +({                                                                     \
 186 +       int __ret = 0;                                                  \
 187 +       if (!(condition))                                               \
 188 +               __swait_event_interruptible(wq, condition, __ret);      \
 189 +       __ret;                                                          \
 190 +})
 191 +
 192 +#define swait_event_interruptible_timeout(wq, condition, timeout)      \
 193 +({                                                                     \
 194 +       int __ret = timeout;                                            \
 195 +       if (!(condition))                                               \
 196 +               __swait_event_interruptible_timeout(wq, condition, __ret);      \
 197 +       __ret;                                                          \
 198 +})
 199 +
 200 +#define __swait_event_timeout(wq, condition, ret)                      \
 201 +do {                                                                   \
 202 +       DEFINE_SWAITER(__wait);                                         \
 203 +                                                                       \
 204 +       for (;;) {                                                      \
 205 +               swait_prepare(&wq, &__wait, TASK_UNINTERRUPTIBLE);      \
 206 +               if (condition)                                          \
 207 +                       break;                                          \
 208 +               ret = schedule_timeout(ret);                            \
 209 +               if (!ret)                                               \
 210 +                       break;                                          \
 211 +       }                                                               \
 212 +       swait_finish(&wq, &__wait);                                     \
 213 +} while (0)
 214 +
 215 +/**
 216 + * swait_event_timeout - sleep until a condition gets true or a timeout elapses
 217 + * @wq: the waitqueue to wait on
 218 + * @condition: a C expression for the event to wait for
 219 + * @timeout: timeout, in jiffies
 220 + *
 221 + * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 222 + * @condition evaluates to true. The @condition is checked each time
 223 + * the waitqueue @wq is woken up.
 224 + *
 225 + * wake_up() has to be called after changing any variable that could
 226 + * change the result of the wait condition.
 227 + *
 228 + * The function returns 0 if the @timeout elapsed, and the remaining
 229 + * jiffies if the condition evaluated to true before the timeout elapsed.
 230 + */
 231 +#define swait_event_timeout(wq, condition, timeout)                    \
 232 +({                                                                     \
 233 +       long __ret = timeout;                                           \
 234 +       if (!(condition))                                               \
 235 +               __swait_event_timeout(wq, condition, __ret);            \
 236 +       __ret;                                                          \
 237 +})
 238 +
 239 +#endif
 240 diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile
 241 index 43fb319c..26505c1 100644
 242 --- a/kernel/sched/Makefile
 243 +++ b/kernel/sched/Makefile
 244 @@ -15,7 +15,7 @@ endif
 245
 246  obj-y += core.o loadavg.o clock.o cputime.o
 247  obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o
 248 -obj-y += wait.o completion.o idle.o
 249 +obj-y += wait.o wait-simple.o completion.o idle.o
 250  obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o
 251  obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o
 252  obj-$(CONFIG_SCHEDSTATS) += stats.o
 253 diff --git a/kernel/sched/wait-simple.c b/kernel/sched/wait-simple.c
 254 new file mode 100644
 255 index 0000000..7dfa86d
 256 --- /dev/null
 257 +++ b/kernel/sched/wait-simple.c
 258 @@ -0,0 +1,115 @@
 259 +/*
 260 + * Simple waitqueues without fancy flags and callbacks
 261 + *
 262 + * (C) 2011 Thomas Gleixner <tglx@linutronix.de>
 263 + *
 264 + * Based on kernel/wait.c
 265 + *
 266 + * For licencing details see kernel-base/COPYING
 267 + */
 268 +#include <linux/init.h>
 269 +#include <linux/export.h>
 270 +#include <linux/sched.h>
 271 +#include <linux/wait-simple.h>
 272 +
 273 +/* Adds w to head->list. Must be called with head->lock locked. */
 274 +static inline void __swait_enqueue(struct swait_head *head, struct swaiter *w)
 275 +{
 276 +       list_add(&w->node, &head->list);
 277 +       /* We can't let the condition leak before the setting of head */
 278 +       smp_mb();
 279 +}
 280 +
 281 +/* Removes w from head->list. Must be called with head->lock locked. */
 282 +static inline void __swait_dequeue(struct swaiter *w)
 283 +{
 284 +       list_del_init(&w->node);
 285 +}
 286 +
 287 +void __init_swait_head(struct swait_head *head, struct lock_class_key *key)
 288 +{
 289 +       raw_spin_lock_init(&head->lock);
 290 +       lockdep_set_class(&head->lock, key);
 291 +       INIT_LIST_HEAD(&head->list);
 292 +}
 293 +EXPORT_SYMBOL(__init_swait_head);
 294 +
 295 +void swait_prepare_locked(struct swait_head *head, struct swaiter *w)
 296 +{
 297 +       w->task = current;
 298 +       if (list_empty(&w->node))
 299 +               __swait_enqueue(head, w);
 300 +}
 301 +
 302 +void swait_prepare(struct swait_head *head, struct swaiter *w, int state)
 303 +{
 304 +       unsigned long flags;
 305 +
 306 +       raw_spin_lock_irqsave(&head->lock, flags);
 307 +       swait_prepare_locked(head, w);
 308 +       __set_current_state(state);
 309 +       raw_spin_unlock_irqrestore(&head->lock, flags);
 310 +}
 311 +EXPORT_SYMBOL(swait_prepare);
 312 +
 313 +void swait_finish_locked(struct swait_head *head, struct swaiter *w)
 314 +{
 315 +       __set_current_state(TASK_RUNNING);
 316 +       if (w->task)
 317 +               __swait_dequeue(w);
 318 +}
 319 +
 320 +void swait_finish(struct swait_head *head, struct swaiter *w)
 321 +{
 322 +       unsigned long flags;
 323 +
 324 +       __set_current_state(TASK_RUNNING);
 325 +       if (w->task) {
 326 +               raw_spin_lock_irqsave(&head->lock, flags);
 327 +               __swait_dequeue(w);
 328 +               raw_spin_unlock_irqrestore(&head->lock, flags);
 329 +       }
 330 +}
 331 +EXPORT_SYMBOL(swait_finish);
 332 +
 333 +unsigned int
 334 +__swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num)
 335 +{
 336 +       struct swaiter *curr, *next;
 337 +       int woken = 0;
 338 +
 339 +       list_for_each_entry_safe(curr, next, &head->list, node) {
 340 +               if (wake_up_state(curr->task, state)) {
 341 +                       __swait_dequeue(curr);
 342 +                       /*
 343 +                        * The waiting task can free the waiter as
 344 +                        * soon as curr->task = NULL is written,
 345 +                        * without taking any locks. A memory barrier
 346 +                        * is required here to prevent the following
 347 +                        * store to curr->task from getting ahead of
 348 +                        * the dequeue operation.
 349 +                        */
 350 +                       smp_wmb();
 351 +                       curr->task = NULL;
 352 +                       if (++woken == num)
 353 +                               break;
 354 +               }
 355 +       }
 356 +       return woken;
 357 +}
 358 +
 359 +unsigned int
 360 +__swait_wake(struct swait_head *head, unsigned int state, unsigned int num)
 361 +{
 362 +       unsigned long flags;
 363 +       int woken;
 364 +
 365 +       if (!swaitqueue_active(head))
 366 +               return 0;
 367 +
 368 +       raw_spin_lock_irqsave(&head->lock, flags);
 369 +       woken = __swait_wake_locked(head, state, num);
 370 +       raw_spin_unlock_irqrestore(&head->lock, flags);
 371 +       return woken;
 372 +}
 373 +EXPORT_SYMBOL(__swait_wake);
 374 --
 375 1.9.1
 376