Apply preempt_rt patch-4.9-rt1.patch.xz

[zynq/linux.git] / kernel / irq_work.c

diff --git a/kernel/irq_work.c b/kernel/irq_work.c
index bcf107ce085450552c17d6b045816cd4656e97c0..2899ba0d23d1755926c87d21d842c8cdc0ec2623 100644 (file)
--- a/kernel/irq_work.c
+++ b/kernel/irq_work.c
@@ -17,6 +17,7 @@
 #include <linux/cpu.h>
 #include <linux/notifier.h>
 #include <linux/smp.h>
+#include <linux/interrupt.h>
 #include <asm/processor.h>
 
 
@@ -65,6 +66,8 @@ void __weak arch_irq_work_raise(void)
  */
 bool irq_work_queue_on(struct irq_work *work, int cpu)
 {
+       struct llist_head *list;
+
        /* All work should have been flushed before going offline */
        WARN_ON_ONCE(cpu_is_offline(cpu));
 
@@ -75,7 +78,12 @@ bool irq_work_queue_on(struct irq_work *work, int cpu)
        if (!irq_work_claim(work))
                return false;
 
-       if (llist_add(&work->llnode, &per_cpu(raised_list, cpu)))
+       if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL) && !(work->flags & IRQ_WORK_HARD_IRQ))
+               list = &per_cpu(lazy_list, cpu);
+       else
+               list = &per_cpu(raised_list, cpu);
+
+       if (llist_add(&work->llnode, list))
                arch_send_call_function_single_ipi(cpu);
 
        return true;
@@ -86,6 +94,9 @@ EXPORT_SYMBOL_GPL(irq_work_queue_on);
 /* Enqueue the irq work @work on the current CPU */
 bool irq_work_queue(struct irq_work *work)
 {
+       struct llist_head *list;
+       bool lazy_work, realtime = IS_ENABLED(CONFIG_PREEMPT_RT_FULL);
+
        /* Only queue if not already pending */
        if (!irq_work_claim(work))
                return false;
@@ -93,13 +104,15 @@ bool irq_work_queue(struct irq_work *work)
        /* Queue the entry and raise the IPI if needed. */
        preempt_disable();
 
-       /* If the work is "lazy", handle it from next tick if any */
-       if (work->flags & IRQ_WORK_LAZY) {
-               if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) &&
-                   tick_nohz_tick_stopped())
-                       arch_irq_work_raise();
-       } else {
-               if (llist_add(&work->llnode, this_cpu_ptr(&raised_list)))
+       lazy_work = work->flags & IRQ_WORK_LAZY;
+
+       if (lazy_work || (realtime && !(work->flags & IRQ_WORK_HARD_IRQ)))
+               list = this_cpu_ptr(&lazy_list);
+       else
+               list = this_cpu_ptr(&raised_list);
+
+       if (llist_add(&work->llnode, list)) {
+               if (!lazy_work || tick_nohz_tick_stopped())
                        arch_irq_work_raise();
        }
 
@@ -116,9 +129,8 @@ bool irq_work_needs_cpu(void)
        raised = this_cpu_ptr(&raised_list);
        lazy = this_cpu_ptr(&lazy_list);
 
-       if (llist_empty(raised) || arch_irq_work_has_interrupt())
-               if (llist_empty(lazy))
-                       return false;
+       if (llist_empty(raised) && llist_empty(lazy))
+               return false;
 
        /* All work should have been flushed before going offline */
        WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
@@ -132,7 +144,7 @@ static void irq_work_run_list(struct llist_head *list)
        struct irq_work *work;
        struct llist_node *llnode;
 
-       BUG_ON(!irqs_disabled());
+       BUG_ON_NONRT(!irqs_disabled());
 
        if (llist_empty(list))
                return;
@@ -169,7 +181,16 @@ static void irq_work_run_list(struct llist_head *list)
 void irq_work_run(void)
 {
        irq_work_run_list(this_cpu_ptr(&raised_list));
-       irq_work_run_list(this_cpu_ptr(&lazy_list));
+       if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL)) {
+               /*
+                * NOTE: we raise softirq via IPI for safety,
+                * and execute in irq_work_tick() to move the
+                * overhead from hard to soft irq context.
+                */
+               if (!llist_empty(this_cpu_ptr(&lazy_list)))
+                       raise_softirq(TIMER_SOFTIRQ);
+       } else
+               irq_work_run_list(this_cpu_ptr(&lazy_list));
 }
 EXPORT_SYMBOL_GPL(irq_work_run);
 
@@ -179,8 +200,17 @@ void irq_work_tick(void)
 
        if (!llist_empty(raised) && !arch_irq_work_has_interrupt())
                irq_work_run_list(raised);
+
+       if (!IS_ENABLED(CONFIG_PREEMPT_RT_FULL))
+               irq_work_run_list(this_cpu_ptr(&lazy_list));
+}
+
+#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_PREEMPT_RT_FULL)
+void irq_work_tick_soft(void)
+{
        irq_work_run_list(this_cpu_ptr(&lazy_list));
 }
+#endif
 
 /*
  * Synchronize against the irq_work @entry, ensures the entry is not