rcu: Make ksoftirqd do RCU quiescent states

Implementing RCU-bh in terms of RCU-preempt makes the system vulnerable
to network-based denial-of-service attacks.  This patch therefore
makes __do_softirq() invoke rcu_bh_qs(), but only when __do_softirq()
is running in ksoftirqd context.  A wrapper layer in interposed so that
other calls to __do_softirq() avoid invoking rcu_bh_qs().  The underlying
function __do_softirq_common() does the actual work.

The reason that rcu_bh_qs() is bad in these non-ksoftirqd contexts is
that there might be a local_bh_enable() inside an RCU-preempt read-side
critical section.  This local_bh_enable() can invoke __do_softirq()
directly, so if __do_softirq() were to invoke rcu_bh_qs() (which just
calls rcu_preempt_qs() in the PREEMPT_RT_FULL case), there would be
an illegal RCU-preempt quiescent state in the middle of an RCU-preempt
read-side critical section.  Therefore, quiescent states can only happen
in cases where __do_softirq() is invoked directly from ksoftirqd.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Link: http://lkml.kernel.org/r/20111005184518.GA21601@linux.vnet.ibm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 0cdb243737440cfbe8f0e7dd9d0969fb69c0b624..09939548dc160f2c45f51d08537c2969aa915fca 100644 (file)
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -270,13 +270,7 @@ static inline int rcu_preempt_depth(void)
 /* Internal to kernel */
 void rcu_init(void);
 void rcu_sched_qs(void);
-
-#ifdef CONFIG_PREEMPT_RT_FULL
-static inline void rcu_bh_qs(void) { }
-#else
 void rcu_bh_qs(void);
-#endif
-
 void rcu_check_callbacks(int user);
 struct notifier_block;
 void rcu_idle_enter(void);

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index bb8f35762bbba69e43b9d17dd5bb05836eb81b65..64ccd648ced29bc4c058029e9dd4bd7ff7c2500e 100644 (file)
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -198,7 +198,14 @@ void rcu_sched_qs(void)
        }
 }
 
-#ifndef CONFIG_PREEMPT_RT_FULL
+#ifdef CONFIG_PREEMPT_RT_FULL
+static void rcu_preempt_qs(void);
+
+void rcu_bh_qs(void)
+{
+       rcu_preempt_qs();
+}
+#else
 void rcu_bh_qs(void)
 {
        if (!__this_cpu_read(rcu_bh_data.passed_quiesce)) {

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index ba7a1603365dd4c7517119b67c30e4074871d98b..49a792b9cacfc8bceeb030ae3e4c75d694ea6885 100644 (file)
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -28,6 +28,7 @@
 #include <linux/gfp.h>
 #include <linux/oom.h>
 #include <linux/smpboot.h>
+#include <linux/jiffies.h>
 #include "../time/tick-internal.h"
 
 #ifdef CONFIG_RCU_BOOST
@@ -1360,7 +1361,7 @@ static void rcu_prepare_kthreads(int cpu)
 
 #endif /* #else #ifdef CONFIG_RCU_BOOST */
 
-#if !defined(CONFIG_RCU_FAST_NO_HZ)
+#if !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL)
 
 /*
  * Check to see if any future RCU-related work will need to be done
@@ -1378,7 +1379,9 @@ int rcu_needs_cpu(unsigned long *delta_jiffies)
        return rcu_cpu_has_callbacks(NULL);
 }
 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
+#endif /* !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL) */
 
+#if !defined(CONFIG_RCU_FAST_NO_HZ)
 /*
  * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
  * after it.
@@ -1476,6 +1479,8 @@ static bool __maybe_unused rcu_try_advance_all_cbs(void)
        return cbs_ready;
 }
 
+#ifndef CONFIG_PREEMPT_RT_FULL
+
 /*
  * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready
  * to invoke.  If the CPU has callbacks, try to advance them.  Tell the
@@ -1516,7 +1521,7 @@ int rcu_needs_cpu(unsigned long *dj)
        return 0;
 }
 #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
-
+#endif /* #ifndef CONFIG_PREEMPT_RT_FULL */
 /*
  * Prepare a CPU for idle from an RCU perspective.  The first major task
  * is to sense whether nohz mode has been enabled or disabled via sysfs.

diff --git a/kernel/softirq.c b/kernel/softirq.c
index a3ee908414a843901fba21610ab3f9b6c5544d40..bae0987164a7a363213222bc1a2d282a4351a022 100644 (file)
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -142,7 +142,7 @@ static void wakeup_softirqd(void)
                wake_up_process(tsk);
 }
 
-static void handle_pending_softirqs(u32 pending)
+static void handle_pending_softirqs(u32 pending, int need_rcu_bh_qs)
 {
        struct softirq_action *h = softirq_vec;
        int softirq_bit;
@@ -175,7 +175,8 @@ static void handle_pending_softirqs(u32 pending)
                pending >>= softirq_bit;
        }
 
-       rcu_bh_qs();
+       if (need_rcu_bh_qs)
+               rcu_bh_qs();
        local_irq_disable();
 }
 
@@ -355,7 +356,7 @@ restart:
        /* Reset the pending bitmask before enabling irqs */
        set_softirq_pending(0);
 
-       handle_pending_softirqs(pending);
+       handle_pending_softirqs(pending, 1);
 
        pending = local_softirq_pending();
        if (pending) {
@@ -404,7 +405,12 @@ static void ksoftirqd_clr_sched_params(unsigned int cpu, bool online) { }
 static DEFINE_LOCAL_IRQ_LOCK(local_softirq_lock);
 static DEFINE_PER_CPU(struct task_struct *, local_softirq_runner);
 
-asmlinkage void __do_softirq(void);
+static void __do_softirq_common(int need_rcu_bh_qs);
+
+asmlinkage __do_softirq(void)
+{
+       __do_softirq_common(0);
+}
 
 void __init softirq_early_init(void)
 {
@@ -513,7 +519,7 @@ asmlinkage void __do_softirq(void)
 
        lockdep_softirq_enter();
 
-       handle_pending_softirqs(pending, cpu);
+       handle_pending_softirqs(pending, need_rcu_bh_qs);
 
        pending = local_softirq_pending();
        if (pending)
@@ -552,7 +558,7 @@ static int __thread_do_softirq(int cpu)
         * schedule!
         */
        if (local_softirq_pending())
-               __do_softirq();
+               __do_softirq_common(cpu >= 0);
        local_unlock(local_softirq_lock);
        unpin_current_cpu();
        preempt_disable();