1
2#define pr_fmt(fmt) "%s: " fmt, __func__
3
4#include <linux/kernel.h>
5#include <linux/sched.h>
6#include <linux/wait.h>
7#include <linux/slab.h>
8#include <linux/mm.h>
9#include <linux/percpu-refcount.h>
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37#define PERCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1))
38
39static DEFINE_SPINLOCK(percpu_ref_switch_lock);
40static DECLARE_WAIT_QUEUE_HEAD(percpu_ref_switch_waitq);
41
42static unsigned long __percpu *percpu_count_ptr(struct percpu_ref *ref)
43{
44 return (unsigned long __percpu *)
45 (ref->percpu_count_ptr & ~__PERCPU_REF_ATOMIC_DEAD);
46}
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release,
64 unsigned int flags, gfp_t gfp)
65{
66 size_t align = max_t(size_t, 1 << __PERCPU_REF_FLAG_BITS,
67 __alignof__(unsigned long));
68 unsigned long start_count = 0;
69 struct percpu_ref_data *data;
70
71 ref->percpu_count_ptr = (unsigned long)
72 __alloc_percpu_gfp(sizeof(unsigned long), align, gfp);
73 if (!ref->percpu_count_ptr)
74 return -ENOMEM;
75
76 data = kzalloc(sizeof(*ref->data), gfp);
77 if (!data) {
78 free_percpu((void __percpu *)ref->percpu_count_ptr);
79 return -ENOMEM;
80 }
81
82 data->force_atomic = flags & PERCPU_REF_INIT_ATOMIC;
83 data->allow_reinit = flags & PERCPU_REF_ALLOW_REINIT;
84
85 if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD)) {
86 ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC;
87 data->allow_reinit = true;
88 } else {
89 start_count += PERCPU_COUNT_BIAS;
90 }
91
92 if (flags & PERCPU_REF_INIT_DEAD)
93 ref->percpu_count_ptr |= __PERCPU_REF_DEAD;
94 else
95 start_count++;
96
97 atomic_long_set(&data->count, start_count);
98
99 data->release = release;
100 data->confirm_switch = NULL;
101 data->ref = ref;
102 ref->data = data;
103 return 0;
104}
105EXPORT_SYMBOL_GPL(percpu_ref_init);
106
107static void __percpu_ref_exit(struct percpu_ref *ref)
108{
109 unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
110
111 if (percpu_count) {
112
113 WARN_ON_ONCE(ref->data && ref->data->confirm_switch);
114 free_percpu(percpu_count);
115 ref->percpu_count_ptr = __PERCPU_REF_ATOMIC_DEAD;
116 }
117}
118
119
120
121
122
123
124
125
126
127
128
129void percpu_ref_exit(struct percpu_ref *ref)
130{
131 struct percpu_ref_data *data = ref->data;
132 unsigned long flags;
133
134 __percpu_ref_exit(ref);
135
136 if (!data)
137 return;
138
139 spin_lock_irqsave(&percpu_ref_switch_lock, flags);
140 ref->percpu_count_ptr |= atomic_long_read(&ref->data->count) <<
141 __PERCPU_REF_FLAG_BITS;
142 ref->data = NULL;
143 spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
144
145 kfree(data);
146}
147EXPORT_SYMBOL_GPL(percpu_ref_exit);
148
149static void percpu_ref_call_confirm_rcu(struct rcu_head *rcu)
150{
151 struct percpu_ref_data *data = container_of(rcu,
152 struct percpu_ref_data, rcu);
153 struct percpu_ref *ref = data->ref;
154
155 data->confirm_switch(ref);
156 data->confirm_switch = NULL;
157 wake_up_all(&percpu_ref_switch_waitq);
158
159 if (!data->allow_reinit)
160 __percpu_ref_exit(ref);
161
162
163 percpu_ref_put(ref);
164}
165
166static void percpu_ref_switch_to_atomic_rcu(struct rcu_head *rcu)
167{
168 struct percpu_ref_data *data = container_of(rcu,
169 struct percpu_ref_data, rcu);
170 struct percpu_ref *ref = data->ref;
171 unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
172 static atomic_t underflows;
173 unsigned long count = 0;
174 int cpu;
175
176 for_each_possible_cpu(cpu)
177 count += *per_cpu_ptr(percpu_count, cpu);
178
179 pr_debug("global %lu percpu %lu\n",
180 atomic_long_read(&data->count), count);
181
182
183
184
185
186
187
188
189
190
191
192
193
194 atomic_long_add((long)count - PERCPU_COUNT_BIAS, &data->count);
195
196 if (WARN_ONCE(atomic_long_read(&data->count) <= 0,
197 "percpu ref (%ps) <= 0 (%ld) after switching to atomic",
198 data->release, atomic_long_read(&data->count)) &&
199 atomic_inc_return(&underflows) < 4) {
200 pr_err("%s(): percpu_ref underflow", __func__);
201 mem_dump_obj(data);
202 }
203
204
205 percpu_ref_call_confirm_rcu(rcu);
206}
207
208static void percpu_ref_noop_confirm_switch(struct percpu_ref *ref)
209{
210}
211
212static void __percpu_ref_switch_to_atomic(struct percpu_ref *ref,
213 percpu_ref_func_t *confirm_switch)
214{
215 if (ref->percpu_count_ptr & __PERCPU_REF_ATOMIC) {
216 if (confirm_switch)
217 confirm_switch(ref);
218 return;
219 }
220
221
222 ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC;
223
224
225
226
227
228 ref->data->confirm_switch = confirm_switch ?:
229 percpu_ref_noop_confirm_switch;
230
231 percpu_ref_get(ref);
232 call_rcu(&ref->data->rcu, percpu_ref_switch_to_atomic_rcu);
233}
234
235static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref)
236{
237 unsigned long __percpu *percpu_count = percpu_count_ptr(ref);
238 int cpu;
239
240 BUG_ON(!percpu_count);
241
242 if (!(ref->percpu_count_ptr & __PERCPU_REF_ATOMIC))
243 return;
244
245 if (WARN_ON_ONCE(!ref->data->allow_reinit))
246 return;
247
248 atomic_long_add(PERCPU_COUNT_BIAS, &ref->data->count);
249
250
251
252
253
254
255
256 for_each_possible_cpu(cpu)
257 *per_cpu_ptr(percpu_count, cpu) = 0;
258
259 smp_store_release(&ref->percpu_count_ptr,
260 ref->percpu_count_ptr & ~__PERCPU_REF_ATOMIC);
261}
262
263static void __percpu_ref_switch_mode(struct percpu_ref *ref,
264 percpu_ref_func_t *confirm_switch)
265{
266 struct percpu_ref_data *data = ref->data;
267
268 lockdep_assert_held(&percpu_ref_switch_lock);
269
270
271
272
273
274
275 wait_event_lock_irq(percpu_ref_switch_waitq, !data->confirm_switch,
276 percpu_ref_switch_lock);
277
278 if (data->force_atomic || percpu_ref_is_dying(ref))
279 __percpu_ref_switch_to_atomic(ref, confirm_switch);
280 else
281 __percpu_ref_switch_to_percpu(ref);
282}
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304void percpu_ref_switch_to_atomic(struct percpu_ref *ref,
305 percpu_ref_func_t *confirm_switch)
306{
307 unsigned long flags;
308
309 spin_lock_irqsave(&percpu_ref_switch_lock, flags);
310
311 ref->data->force_atomic = true;
312 __percpu_ref_switch_mode(ref, confirm_switch);
313
314 spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
315}
316EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic);
317
318
319
320
321
322
323
324
325
326void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref)
327{
328 percpu_ref_switch_to_atomic(ref, NULL);
329 wait_event(percpu_ref_switch_waitq, !ref->data->confirm_switch);
330}
331EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic_sync);
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351void percpu_ref_switch_to_percpu(struct percpu_ref *ref)
352{
353 unsigned long flags;
354
355 spin_lock_irqsave(&percpu_ref_switch_lock, flags);
356
357 ref->data->force_atomic = false;
358 __percpu_ref_switch_mode(ref, NULL);
359
360 spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
361}
362EXPORT_SYMBOL_GPL(percpu_ref_switch_to_percpu);
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
382 percpu_ref_func_t *confirm_kill)
383{
384 unsigned long flags;
385
386 spin_lock_irqsave(&percpu_ref_switch_lock, flags);
387
388 WARN_ONCE(percpu_ref_is_dying(ref),
389 "%s called more than once on %ps!", __func__,
390 ref->data->release);
391
392 ref->percpu_count_ptr |= __PERCPU_REF_DEAD;
393 __percpu_ref_switch_mode(ref, confirm_kill);
394 percpu_ref_put(ref);
395
396 spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
397}
398EXPORT_SYMBOL_GPL(percpu_ref_kill_and_confirm);
399
400
401
402
403
404
405
406
407
408bool percpu_ref_is_zero(struct percpu_ref *ref)
409{
410 unsigned long __percpu *percpu_count;
411 unsigned long count, flags;
412
413 if (__ref_is_percpu(ref, &percpu_count))
414 return false;
415
416
417 spin_lock_irqsave(&percpu_ref_switch_lock, flags);
418 if (ref->data)
419 count = atomic_long_read(&ref->data->count);
420 else
421 count = ref->percpu_count_ptr >> __PERCPU_REF_FLAG_BITS;
422 spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
423
424 return count == 0;
425}
426EXPORT_SYMBOL_GPL(percpu_ref_is_zero);
427
428
429
430
431
432
433
434
435
436
437
438
439void percpu_ref_reinit(struct percpu_ref *ref)
440{
441 WARN_ON_ONCE(!percpu_ref_is_zero(ref));
442
443 percpu_ref_resurrect(ref);
444}
445EXPORT_SYMBOL_GPL(percpu_ref_reinit);
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461void percpu_ref_resurrect(struct percpu_ref *ref)
462{
463 unsigned long __percpu *percpu_count;
464 unsigned long flags;
465
466 spin_lock_irqsave(&percpu_ref_switch_lock, flags);
467
468 WARN_ON_ONCE(!percpu_ref_is_dying(ref));
469 WARN_ON_ONCE(__ref_is_percpu(ref, &percpu_count));
470
471 ref->percpu_count_ptr &= ~__PERCPU_REF_DEAD;
472 percpu_ref_get(ref);
473 __percpu_ref_switch_mode(ref, NULL);
474
475 spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
476}
477EXPORT_SYMBOL_GPL(percpu_ref_resurrect);
478