1/* 2 * Read-Copy Update mechanism for mutual exclusion 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License as published by 6 * the Free Software Foundation; either version 2 of the License, or 7 * (at your option) any later version. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program; if not, write to the Free Software 16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 * 18 * Copyright IBM Corporation, 2001 19 * 20 * Authors: Dipankar Sarma <dipankar@in.ibm.com> 21 * Manfred Spraul <manfred@colorfullife.com> 22 * 23 * Based on the original work by Paul McKenney <paulmck@us.ibm.com> 24 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. 25 * Papers: 26 * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf 27 * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001) 28 * 29 * For detailed explanation of Read-Copy Update mechanism see - 30 * http://lse.sourceforge.net/locking/rcupdate.html 31 * 32 */ 33#include <linux/types.h> 34#include <linux/kernel.h> 35#include <linux/init.h> 36#include <linux/spinlock.h> 37#include <linux/smp.h> 38#include <linux/interrupt.h> 39#include <linux/sched.h> 40#include <asm/atomic.h> 41#include <linux/bitops.h> 42#include <linux/percpu.h> 43#include <linux/notifier.h> 44#include <linux/cpu.h> 45#include <linux/mutex.h> 46#include <linux/module.h> 47#include <linux/kernel_stat.h> 48 49#ifdef CONFIG_DEBUG_LOCK_ALLOC 50static struct lock_class_key rcu_lock_key; 51struct lockdep_map rcu_lock_map = 52 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key); 53EXPORT_SYMBOL_GPL(rcu_lock_map); 54#endif 55 56int rcu_scheduler_active __read_mostly; 57 58/* 59 * Awaken the corresponding synchronize_rcu() instance now that a 60 * grace period has elapsed. 61 */ 62void wakeme_after_rcu(struct rcu_head *head) 63{ 64 struct rcu_synchronize *rcu; 65 66 rcu = container_of(head, struct rcu_synchronize, head); 67 complete(&rcu->completion); 68} 69 70#ifdef CONFIG_TREE_PREEMPT_RCU 71 72/** 73 * synchronize_rcu - wait until a grace period has elapsed. 74 * 75 * Control will return to the caller some time after a full grace 76 * period has elapsed, in other words after all currently executing RCU 77 * read-side critical sections have completed. RCU read-side critical 78 * sections are delimited by rcu_read_lock() and rcu_read_unlock(), 79 * and may be nested. 80 */ 81void synchronize_rcu(void) 82{ 83 struct rcu_synchronize rcu; 84 85 if (!rcu_scheduler_active) 86 return; 87 88 init_completion(&rcu.completion); 89 /* Will wake me after RCU finished. */ 90 call_rcu(&rcu.head, wakeme_after_rcu); 91 /* Wait for it. */ 92 wait_for_completion(&rcu.completion); 93} 94EXPORT_SYMBOL_GPL(synchronize_rcu); 95 96#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ 97 98/** 99 * synchronize_sched - wait until an rcu-sched grace period has elapsed. 100 * 101 * Control will return to the caller some time after a full rcu-sched 102 * grace period has elapsed, in other words after all currently executing 103 * rcu-sched read-side critical sections have completed. These read-side 104 * critical sections are delimited by rcu_read_lock_sched() and 105 * rcu_read_unlock_sched(), and may be nested. Note that preempt_disable(), 106 * local_irq_disable(), and so on may be used in place of 107 * rcu_read_lock_sched(). 108 * 109 * This means that all preempt_disable code sequences, including NMI and 110 * hardware-interrupt handlers, in progress on entry will have completed 111 * before this primitive returns. However, this does not guarantee that 112 * softirq handlers will have completed, since in some kernels, these 113 * handlers can run in process context, and can block. 114 * 115 * This primitive provides the guarantees made by the (now removed) 116 * synchronize_kernel() API. In contrast, synchronize_rcu() only 117 * guarantees that rcu_read_lock() sections will have completed. 118 * In "classic RCU", these two guarantees happen to be one and 119 * the same, but can differ in realtime RCU implementations. 120 */ 121void synchronize_sched(void) 122{ 123 struct rcu_synchronize rcu; 124 125 if (rcu_blocking_is_gp()) 126 return; 127 128 init_completion(&rcu.completion); 129 /* Will wake me after RCU finished. */ 130 call_rcu_sched(&rcu.head, wakeme_after_rcu); 131 /* Wait for it. */ 132 wait_for_completion(&rcu.completion); 133} 134EXPORT_SYMBOL_GPL(synchronize_sched); 135 136/** 137 * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed. 138 * 139 * Control will return to the caller some time after a full rcu_bh grace 140 * period has elapsed, in other words after all currently executing rcu_bh 141 * read-side critical sections have completed. RCU read-side critical 142 * sections are delimited by rcu_read_lock_bh() and rcu_read_unlock_bh(), 143 * and may be nested. 144 */ 145void synchronize_rcu_bh(void) 146{ 147 struct rcu_synchronize rcu; 148 149 if (rcu_blocking_is_gp()) 150 return; 151 152 init_completion(&rcu.completion); 153 /* Will wake me after RCU finished. */ 154 call_rcu_bh(&rcu.head, wakeme_after_rcu); 155 /* Wait for it. */ 156 wait_for_completion(&rcu.completion); 157} 158EXPORT_SYMBOL_GPL(synchronize_rcu_bh); 159 160static int __cpuinit rcu_barrier_cpu_hotplug(struct notifier_block *self, 161 unsigned long action, void *hcpu) 162{ 163 return rcu_cpu_notify(self, action, hcpu); 164} 165 166void __init rcu_init(void) 167{ 168 int i; 169 170 __rcu_init(); 171 cpu_notifier(rcu_barrier_cpu_hotplug, 0); 172 173 /* 174 * We don't need protection against CPU-hotplug here because 175 * this is called early in boot, before either interrupts 176 * or the scheduler are operational. 177 */ 178 for_each_online_cpu(i) 179 rcu_barrier_cpu_hotplug(NULL, CPU_UP_PREPARE, (void *)(long)i); 180} 181 182void rcu_scheduler_starting(void) 183{ 184 WARN_ON(num_online_cpus() != 1); 185 WARN_ON(nr_context_switches() > 0); 186 rcu_scheduler_active = 1; 187} 188