Fix bug

[can-eth-gw-linux.git] / kernel / rcutree.h

diff --git a/kernel/rcutree.h b/kernel/rcutree.h
index a240f032848eec8d00e156da54fce3cd24a4b525..4b69291b093d7157d638da01d342c2ee3d1f2d16 100644 (file)
--- a/kernel/rcutree.h
+++ b/kernel/rcutree.h
@@ -287,6 +287,7 @@ struct rcu_data {
        long            qlen_last_fqs_check;
                                        /* qlen at last check for QS forcing */
        unsigned long   n_cbs_invoked;  /* count of RCU cbs invoked. */
+       unsigned long   n_nocbs_invoked; /* count of no-CBs RCU cbs invoked. */
        unsigned long   n_cbs_orphaned; /* RCU cbs orphaned by dying CPU */
        unsigned long   n_cbs_adopted;  /* RCU cbs adopted from dying CPU */
        unsigned long   n_force_qs_snap;
@@ -317,6 +318,18 @@ struct rcu_data {
        struct rcu_head oom_head;
 #endif /* #ifdef CONFIG_RCU_FAST_NO_HZ */
 
+       /* 7) Callback offloading. */
+#ifdef CONFIG_RCU_NOCB_CPU
+       struct rcu_head *nocb_head;     /* CBs waiting for kthread. */
+       struct rcu_head **nocb_tail;
+       atomic_long_t nocb_q_count;     /* # CBs waiting for kthread */
+       atomic_long_t nocb_q_count_lazy; /*  (approximate). */
+       int nocb_p_count;               /* # CBs being invoked by kthread */
+       int nocb_p_count_lazy;          /*  (approximate). */
+       wait_queue_head_t nocb_wq;      /* For nocb kthreads to sleep on. */
+       struct task_struct *nocb_kthread;
+#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
+
        int cpu;
        struct rcu_state *rsp;
 };
@@ -369,6 +382,12 @@ struct rcu_state {
        struct rcu_data __percpu *rda;          /* pointer of percu rcu_data. */
        void (*call)(struct rcu_head *head,     /* call_rcu() flavor. */
                     void (*func)(struct rcu_head *head));
+#ifdef CONFIG_RCU_NOCB_CPU
+       void (*call_remote)(struct rcu_head *head,
+                    void (*func)(struct rcu_head *head));
+                                               /* call_rcu() flavor, but for */
+                                               /*  placing on remote CPU. */
+#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
 
        /* The following fields are guarded by the root rcu_node's lock. */
 
@@ -383,9 +402,8 @@ struct rcu_state {
 
        /* End of fields guarded by root rcu_node's lock. */
 
-       raw_spinlock_t onofflock ____cacheline_internodealigned_in_smp;
-                                               /* exclude on/offline and */
-                                               /*  starting new GP. */
+       raw_spinlock_t orphan_lock ____cacheline_internodealigned_in_smp;
+                                               /* Protect following fields. */
        struct rcu_head *orphan_nxtlist;        /* Orphaned callbacks that */
                                                /*  need a grace period. */
        struct rcu_head **orphan_nxttail;       /* Tail of above. */
@@ -394,7 +412,7 @@ struct rcu_state {
        struct rcu_head **orphan_donetail;      /* Tail of above. */
        long qlen_lazy;                         /* Number of lazy callbacks. */
        long qlen;                              /* Total number of callbacks. */
-       /* End of fields guarded by onofflock. */
+       /* End of fields guarded by orphan_lock. */
 
        struct mutex onoff_mutex;               /* Coordinate hotplug & GPs. */
 
@@ -405,6 +423,18 @@ struct rcu_state {
                                                /*  _rcu_barrier(). */
        /* End of fields guarded by barrier_mutex. */
 
+       atomic_long_t expedited_start;          /* Starting ticket. */
+       atomic_long_t expedited_done;           /* Done ticket. */
+       atomic_long_t expedited_wrap;           /* # near-wrap incidents. */
+       atomic_long_t expedited_tryfail;        /* # acquisition failures. */
+       atomic_long_t expedited_workdone1;      /* # done by others #1. */
+       atomic_long_t expedited_workdone2;      /* # done by others #2. */
+       atomic_long_t expedited_normal;         /* # fallbacks to normal. */
+       atomic_long_t expedited_stoppedcpus;    /* # successful stop_cpus. */
+       atomic_long_t expedited_done_tries;     /* # tries to update _done. */
+       atomic_long_t expedited_done_lost;      /* # times beaten to _done. */
+       atomic_long_t expedited_done_exit;      /* # times exited _done loop. */
+
        unsigned long jiffies_force_qs;         /* Time at which to invoke */
                                                /*  force_quiescent_state(). */
        unsigned long n_force_qs;               /* Number of calls to */
@@ -428,6 +458,8 @@ struct rcu_state {
 #define RCU_GP_FLAG_FQS  0x2   /* Need grace-period quiescent-state forcing. */
 
 extern struct list_head rcu_struct_flavors;
+
+/* Sequence through rcu_state structures for each RCU flavor. */
 #define for_each_rcu_flavor(rsp) \
        list_for_each_entry((rsp), &rcu_struct_flavors, flavors)
 
@@ -504,5 +536,32 @@ static void print_cpu_stall_info(struct rcu_state *rsp, int cpu);
 static void print_cpu_stall_info_end(void);
 static void zero_cpu_stall_ticks(struct rcu_data *rdp);
 static void increment_cpu_stall_ticks(void);
+static bool is_nocb_cpu(int cpu);
+static bool __call_rcu_nocb(struct rcu_data *rdp, struct rcu_head *rhp,
+                           bool lazy);
+static bool rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
+                                     struct rcu_data *rdp);
+static bool nocb_cpu_expendable(int cpu);
+static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp);
+static void rcu_spawn_nocb_kthreads(struct rcu_state *rsp);
+static void init_nocb_callback_list(struct rcu_data *rdp);
+static void __init rcu_init_nocb(void);
 
 #endif /* #ifndef RCU_TREE_NONCORE */
+
+#ifdef CONFIG_RCU_TRACE
+#ifdef CONFIG_RCU_NOCB_CPU
+/* Sum up queue lengths for tracing. */
+static inline void rcu_nocb_q_lengths(struct rcu_data *rdp, long *ql, long *qll)
+{
+       *ql = atomic_long_read(&rdp->nocb_q_count) + rdp->nocb_p_count;
+       *qll = atomic_long_read(&rdp->nocb_q_count_lazy) + rdp->nocb_p_count_lazy;
+}
+#else /* #ifdef CONFIG_RCU_NOCB_CPU */
+static inline void rcu_nocb_q_lengths(struct rcu_data *rdp, long *ql, long *qll)
+{
+       *ql = 0;
+       *qll = 0;
+}
+#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
+#endif /* #ifdef CONFIG_RCU_TRACE */