2 * Sleepable Read-Copy Update mechanism for mutual exclusion
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 * Copyright (C) IBM Corporation, 2006
20 * Author: Paul McKenney <paulmck@us.ibm.com>
22 * For detailed explanation of Read-Copy Update mechanism see -
23 * Documentation/RCU/ *.txt
30 #include <linux/mutex.h>
31 #include <linux/rcupdate.h>
33 struct srcu_struct_array {
40 struct srcu_struct_array __percpu *per_cpu_ref;
42 #ifdef CONFIG_DEBUG_LOCK_ALLOC
43 struct lockdep_map dep_map;
44 #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
47 #ifndef CONFIG_PREEMPT
48 #define srcu_barrier() barrier()
49 #else /* #ifndef CONFIG_PREEMPT */
50 #define srcu_barrier()
51 #endif /* #else #ifndef CONFIG_PREEMPT */
53 #ifdef CONFIG_DEBUG_LOCK_ALLOC
55 int __init_srcu_struct(struct srcu_struct *sp, const char *name,
56 struct lock_class_key *key);
58 #define init_srcu_struct(sp) \
60 static struct lock_class_key __srcu_key; \
62 __init_srcu_struct((sp), #sp, &__srcu_key); \
65 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
67 int init_srcu_struct(struct srcu_struct *sp);
69 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
71 void cleanup_srcu_struct(struct srcu_struct *sp);
72 int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp);
73 void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp);
74 void synchronize_srcu(struct srcu_struct *sp);
75 void synchronize_srcu_expedited(struct srcu_struct *sp);
76 long srcu_batches_completed(struct srcu_struct *sp);
78 #ifdef CONFIG_DEBUG_LOCK_ALLOC
81 * srcu_read_lock_held - might we be in SRCU read-side critical section?
83 * If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU
84 * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC,
85 * this assumes we are in an SRCU read-side critical section unless it can
88 * Checks debug_lockdep_rcu_enabled() to prevent false positives during boot
89 * and while lockdep is disabled.
91 * Note that if the CPU is in the idle loop from an RCU point of view
92 * (ie: that we are in the section between rcu_idle_enter() and
93 * rcu_idle_exit()) then srcu_read_lock_held() returns false even if
94 * the CPU did an srcu_read_lock(). The reason for this is that RCU
95 * ignores CPUs that are in such a section, considering these as in
96 * extended quiescent state, so such a CPU is effectively never in an
97 * RCU read-side critical section regardless of what RCU primitives it
98 * invokes. This state of affairs is required --- we need to keep an
99 * RCU-free window in idle where the CPU may possibly enter into low
100 * power mode. This way we can notice an extended quiescent state to
101 * other CPUs that started a grace period. Otherwise we would delay any
102 * grace period as long as we run in the idle task.
104 * Similarly, we avoid claiming an SRCU read lock held if the current
107 static inline int srcu_read_lock_held(struct srcu_struct *sp)
109 if (!debug_lockdep_rcu_enabled())
111 if (rcu_is_cpu_idle())
113 if (!rcu_lockdep_current_cpu_online())
115 return lock_is_held(&sp->dep_map);
118 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
120 static inline int srcu_read_lock_held(struct srcu_struct *sp)
125 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
128 * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing
129 * @p: the pointer to fetch and protect for later dereferencing
130 * @sp: pointer to the srcu_struct, which is used to check that we
131 * really are in an SRCU read-side critical section.
132 * @c: condition to check for update-side use
134 * If PROVE_RCU is enabled, invoking this outside of an RCU read-side
135 * critical section will result in an RCU-lockdep splat, unless @c evaluates
136 * to 1. The @c argument will normally be a logical expression containing
137 * lockdep_is_held() calls.
139 #define srcu_dereference_check(p, sp, c) \
140 __rcu_dereference_check((p), srcu_read_lock_held(sp) || (c), __rcu)
143 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing
144 * @p: the pointer to fetch and protect for later dereferencing
145 * @sp: pointer to the srcu_struct, which is used to check that we
146 * really are in an SRCU read-side critical section.
148 * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU
149 * is enabled, invoking this outside of an RCU read-side critical
150 * section will result in an RCU-lockdep splat.
152 #define srcu_dereference(p, sp) srcu_dereference_check((p), (sp), 0)
155 * srcu_read_lock - register a new reader for an SRCU-protected structure.
156 * @sp: srcu_struct in which to register the new reader.
158 * Enter an SRCU read-side critical section. Note that SRCU read-side
159 * critical sections may be nested. However, it is illegal to
160 * call anything that waits on an SRCU grace period for the same
161 * srcu_struct, whether directly or indirectly. Please note that
162 * one way to indirectly wait on an SRCU grace period is to acquire
163 * a mutex that is held elsewhere while calling synchronize_srcu() or
164 * synchronize_srcu_expedited().
166 * Note that srcu_read_lock() and the matching srcu_read_unlock() must
167 * occur in the same context, for example, it is illegal to invoke
168 * srcu_read_unlock() in an irq handler if the matching srcu_read_lock()
169 * was invoked in process context.
171 static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp)
173 int retval = __srcu_read_lock(sp);
175 rcu_lock_acquire(&(sp)->dep_map);
176 rcu_lockdep_assert(!rcu_is_cpu_idle(),
177 "srcu_read_lock() used illegally while idle");
182 * srcu_read_unlock - unregister a old reader from an SRCU-protected structure.
183 * @sp: srcu_struct in which to unregister the old reader.
184 * @idx: return value from corresponding srcu_read_lock().
186 * Exit an SRCU read-side critical section.
188 static inline void srcu_read_unlock(struct srcu_struct *sp, int idx)
191 rcu_lockdep_assert(!rcu_is_cpu_idle(),
192 "srcu_read_unlock() used illegally while idle");
193 rcu_lock_release(&(sp)->dep_map);
194 __srcu_read_unlock(sp, idx);
198 * srcu_read_lock_raw - register a new reader for an SRCU-protected structure.
199 * @sp: srcu_struct in which to register the new reader.
201 * Enter an SRCU read-side critical section. Similar to srcu_read_lock(),
202 * but avoids the RCU-lockdep checking. This means that it is legal to
203 * use srcu_read_lock_raw() in one context, for example, in an exception
204 * handler, and then have the matching srcu_read_unlock_raw() in another
205 * context, for example in the task that took the exception.
207 * However, the entire SRCU read-side critical section must reside within a
208 * single task. For example, beware of using srcu_read_lock_raw() in
209 * a device interrupt handler and srcu_read_unlock() in the interrupted
210 * task: This will not work if interrupts are threaded.
212 static inline int srcu_read_lock_raw(struct srcu_struct *sp)
217 local_irq_save(flags);
218 ret = __srcu_read_lock(sp);
219 local_irq_restore(flags);
224 * srcu_read_unlock_raw - unregister reader from an SRCU-protected structure.
225 * @sp: srcu_struct in which to unregister the old reader.
226 * @idx: return value from corresponding srcu_read_lock_raw().
228 * Exit an SRCU read-side critical section without lockdep-RCU checking.
229 * See srcu_read_lock_raw() for more details.
231 static inline void srcu_read_unlock_raw(struct srcu_struct *sp, int idx)
235 local_irq_save(flags);
236 __srcu_read_unlock(sp, idx);
237 local_irq_restore(flags);