1
2
3
4
5
6
7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9#include <linux/sched.h>
10#include <linux/slab.h>
11#include <linux/spinlock.h>
12#include <linux/buffer_head.h>
13#include <linux/delay.h>
14#include <linux/sort.h>
15#include <linux/hash.h>
16#include <linux/jhash.h>
17#include <linux/kallsyms.h>
18#include <linux/gfs2_ondisk.h>
19#include <linux/list.h>
20#include <linux/wait.h>
21#include <linux/module.h>
22#include <linux/uaccess.h>
23#include <linux/seq_file.h>
24#include <linux/debugfs.h>
25#include <linux/kthread.h>
26#include <linux/freezer.h>
27#include <linux/workqueue.h>
28#include <linux/jiffies.h>
29#include <linux/rcupdate.h>
30#include <linux/rculist_bl.h>
31#include <linux/bit_spinlock.h>
32#include <linux/percpu.h>
33#include <linux/list_sort.h>
34#include <linux/lockref.h>
35#include <linux/rhashtable.h>
36
37#include "gfs2.h"
38#include "incore.h"
39#include "glock.h"
40#include "glops.h"
41#include "inode.h"
42#include "lops.h"
43#include "meta_io.h"
44#include "quota.h"
45#include "super.h"
46#include "util.h"
47#include "bmap.h"
48#define CREATE_TRACE_POINTS
49#include "trace_gfs2.h"
50
51struct gfs2_glock_iter {
52 struct gfs2_sbd *sdp;
53 struct rhashtable_iter hti;
54 struct gfs2_glock *gl;
55 loff_t last_pos;
56};
57
58typedef void (*glock_examiner) (struct gfs2_glock * gl);
59
60static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
61
62static struct dentry *gfs2_root;
63static struct workqueue_struct *glock_workqueue;
64struct workqueue_struct *gfs2_delete_workqueue;
65static LIST_HEAD(lru_list);
66static atomic_t lru_count = ATOMIC_INIT(0);
67static DEFINE_SPINLOCK(lru_lock);
68
69#define GFS2_GL_HASH_SHIFT 15
70#define GFS2_GL_HASH_SIZE BIT(GFS2_GL_HASH_SHIFT)
71
72static const struct rhashtable_params ht_parms = {
73 .nelem_hint = GFS2_GL_HASH_SIZE * 3 / 4,
74 .key_len = offsetofend(struct lm_lockname, ln_type),
75 .key_offset = offsetof(struct gfs2_glock, gl_name),
76 .head_offset = offsetof(struct gfs2_glock, gl_node),
77};
78
79static struct rhashtable gl_hash_table;
80
81#define GLOCK_WAIT_TABLE_BITS 12
82#define GLOCK_WAIT_TABLE_SIZE (1 << GLOCK_WAIT_TABLE_BITS)
83static wait_queue_head_t glock_wait_table[GLOCK_WAIT_TABLE_SIZE] __cacheline_aligned;
84
85struct wait_glock_queue {
86 struct lm_lockname *name;
87 wait_queue_entry_t wait;
88};
89
90static int glock_wake_function(wait_queue_entry_t *wait, unsigned int mode,
91 int sync, void *key)
92{
93 struct wait_glock_queue *wait_glock =
94 container_of(wait, struct wait_glock_queue, wait);
95 struct lm_lockname *wait_name = wait_glock->name;
96 struct lm_lockname *wake_name = key;
97
98 if (wake_name->ln_sbd != wait_name->ln_sbd ||
99 wake_name->ln_number != wait_name->ln_number ||
100 wake_name->ln_type != wait_name->ln_type)
101 return 0;
102 return autoremove_wake_function(wait, mode, sync, key);
103}
104
105static wait_queue_head_t *glock_waitqueue(struct lm_lockname *name)
106{
107 u32 hash = jhash2((u32 *)name, ht_parms.key_len / 4, 0);
108
109 return glock_wait_table + hash_32(hash, GLOCK_WAIT_TABLE_BITS);
110}
111
112
113
114
115
116static void wake_up_glock(struct gfs2_glock *gl)
117{
118 wait_queue_head_t *wq = glock_waitqueue(&gl->gl_name);
119
120 if (waitqueue_active(wq))
121 __wake_up(wq, TASK_NORMAL, 1, &gl->gl_name);
122}
123
124static void gfs2_glock_dealloc(struct rcu_head *rcu)
125{
126 struct gfs2_glock *gl = container_of(rcu, struct gfs2_glock, gl_rcu);
127
128 if (gl->gl_ops->go_flags & GLOF_ASPACE) {
129 kmem_cache_free(gfs2_glock_aspace_cachep, gl);
130 } else {
131 kfree(gl->gl_lksb.sb_lvbptr);
132 kmem_cache_free(gfs2_glock_cachep, gl);
133 }
134}
135
136void gfs2_glock_free(struct gfs2_glock *gl)
137{
138 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
139
140 BUG_ON(atomic_read(&gl->gl_revokes));
141 rhashtable_remove_fast(&gl_hash_table, &gl->gl_node, ht_parms);
142 smp_mb();
143 wake_up_glock(gl);
144 call_rcu(&gl->gl_rcu, gfs2_glock_dealloc);
145 if (atomic_dec_and_test(&sdp->sd_glock_disposal))
146 wake_up(&sdp->sd_glock_wait);
147}
148
149
150
151
152
153
154
155void gfs2_glock_hold(struct gfs2_glock *gl)
156{
157 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
158 lockref_get(&gl->gl_lockref);
159}
160
161
162
163
164
165
166
167
168static int demote_ok(const struct gfs2_glock *gl)
169{
170 const struct gfs2_glock_operations *glops = gl->gl_ops;
171
172 if (gl->gl_state == LM_ST_UNLOCKED)
173 return 0;
174 if (!list_empty(&gl->gl_holders))
175 return 0;
176 if (glops->go_demote_ok)
177 return glops->go_demote_ok(gl);
178 return 1;
179}
180
181
182void gfs2_glock_add_to_lru(struct gfs2_glock *gl)
183{
184 if (!(gl->gl_ops->go_flags & GLOF_LRU))
185 return;
186
187 spin_lock(&lru_lock);
188
189 list_del(&gl->gl_lru);
190 list_add_tail(&gl->gl_lru, &lru_list);
191
192 if (!test_bit(GLF_LRU, &gl->gl_flags)) {
193 set_bit(GLF_LRU, &gl->gl_flags);
194 atomic_inc(&lru_count);
195 }
196
197 spin_unlock(&lru_lock);
198}
199
200static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl)
201{
202 if (!(gl->gl_ops->go_flags & GLOF_LRU))
203 return;
204
205 spin_lock(&lru_lock);
206 if (test_bit(GLF_LRU, &gl->gl_flags)) {
207 list_del_init(&gl->gl_lru);
208 atomic_dec(&lru_count);
209 clear_bit(GLF_LRU, &gl->gl_flags);
210 }
211 spin_unlock(&lru_lock);
212}
213
214
215
216
217
218static void __gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
219 if (!queue_delayed_work(glock_workqueue, &gl->gl_work, delay)) {
220
221
222
223
224
225
226 GLOCK_BUG_ON(gl, gl->gl_lockref.count < 2);
227 gl->gl_lockref.count--;
228 }
229}
230
231static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
232 spin_lock(&gl->gl_lockref.lock);
233 __gfs2_glock_queue_work(gl, delay);
234 spin_unlock(&gl->gl_lockref.lock);
235}
236
237static void __gfs2_glock_put(struct gfs2_glock *gl)
238{
239 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
240 struct address_space *mapping = gfs2_glock2aspace(gl);
241
242 lockref_mark_dead(&gl->gl_lockref);
243
244 gfs2_glock_remove_from_lru(gl);
245 spin_unlock(&gl->gl_lockref.lock);
246 GLOCK_BUG_ON(gl, !list_empty(&gl->gl_holders));
247 GLOCK_BUG_ON(gl, mapping && mapping->nrpages);
248 trace_gfs2_glock_put(gl);
249 sdp->sd_lockstruct.ls_ops->lm_put_lock(gl);
250}
251
252
253
254
255void gfs2_glock_queue_put(struct gfs2_glock *gl)
256{
257 gfs2_glock_queue_work(gl, 0);
258}
259
260
261
262
263
264
265
266void gfs2_glock_put(struct gfs2_glock *gl)
267{
268 if (lockref_put_or_lock(&gl->gl_lockref))
269 return;
270
271 __gfs2_glock_put(gl);
272}
273
274
275
276
277
278
279
280
281
282static inline int may_grant(const struct gfs2_glock *gl, const struct gfs2_holder *gh)
283{
284 const struct gfs2_holder *gh_head = list_entry(gl->gl_holders.next, const struct gfs2_holder, gh_list);
285 if ((gh->gh_state == LM_ST_EXCLUSIVE ||
286 gh_head->gh_state == LM_ST_EXCLUSIVE) && gh != gh_head)
287 return 0;
288 if (gl->gl_state == gh->gh_state)
289 return 1;
290 if (gh->gh_flags & GL_EXACT)
291 return 0;
292 if (gl->gl_state == LM_ST_EXCLUSIVE) {
293 if (gh->gh_state == LM_ST_SHARED && gh_head->gh_state == LM_ST_SHARED)
294 return 1;
295 if (gh->gh_state == LM_ST_DEFERRED && gh_head->gh_state == LM_ST_DEFERRED)
296 return 1;
297 }
298 if (gl->gl_state != LM_ST_UNLOCKED && (gh->gh_flags & LM_FLAG_ANY))
299 return 1;
300 return 0;
301}
302
303static void gfs2_holder_wake(struct gfs2_holder *gh)
304{
305 clear_bit(HIF_WAIT, &gh->gh_iflags);
306 smp_mb__after_atomic();
307 wake_up_bit(&gh->gh_iflags, HIF_WAIT);
308}
309
310
311
312
313
314
315static void do_error(struct gfs2_glock *gl, const int ret)
316{
317 struct gfs2_holder *gh, *tmp;
318
319 list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
320 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
321 continue;
322 if (ret & LM_OUT_ERROR)
323 gh->gh_error = -EIO;
324 else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))
325 gh->gh_error = GLR_TRYFAILED;
326 else
327 continue;
328 list_del_init(&gh->gh_list);
329 trace_gfs2_glock_queue(gh, 0);
330 gfs2_holder_wake(gh);
331 }
332}
333
334
335
336
337
338
339
340
341
342static int do_promote(struct gfs2_glock *gl)
343__releases(&gl->gl_lockref.lock)
344__acquires(&gl->gl_lockref.lock)
345{
346 const struct gfs2_glock_operations *glops = gl->gl_ops;
347 struct gfs2_holder *gh, *tmp;
348 int ret;
349
350restart:
351 list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
352 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
353 continue;
354 if (may_grant(gl, gh)) {
355 if (gh->gh_list.prev == &gl->gl_holders &&
356 glops->go_lock) {
357 spin_unlock(&gl->gl_lockref.lock);
358
359 ret = glops->go_lock(gh);
360 spin_lock(&gl->gl_lockref.lock);
361 if (ret) {
362 if (ret == 1)
363 return 2;
364 gh->gh_error = ret;
365 list_del_init(&gh->gh_list);
366 trace_gfs2_glock_queue(gh, 0);
367 gfs2_holder_wake(gh);
368 goto restart;
369 }
370 set_bit(HIF_HOLDER, &gh->gh_iflags);
371 trace_gfs2_promote(gh, 1);
372 gfs2_holder_wake(gh);
373 goto restart;
374 }
375 set_bit(HIF_HOLDER, &gh->gh_iflags);
376 trace_gfs2_promote(gh, 0);
377 gfs2_holder_wake(gh);
378 continue;
379 }
380 if (gh->gh_list.prev == &gl->gl_holders)
381 return 1;
382 do_error(gl, 0);
383 break;
384 }
385 return 0;
386}
387
388
389
390
391
392
393static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl)
394{
395 struct gfs2_holder *gh;
396
397 list_for_each_entry(gh, &gl->gl_holders, gh_list) {
398 if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
399 return gh;
400 }
401 return NULL;
402}
403
404
405
406
407
408
409
410
411static void state_change(struct gfs2_glock *gl, unsigned int new_state)
412{
413 int held1, held2;
414
415 held1 = (gl->gl_state != LM_ST_UNLOCKED);
416 held2 = (new_state != LM_ST_UNLOCKED);
417
418 if (held1 != held2) {
419 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
420 if (held2)
421 gl->gl_lockref.count++;
422 else
423 gl->gl_lockref.count--;
424 }
425 if (held1 && held2 && list_empty(&gl->gl_holders))
426 clear_bit(GLF_QUEUED, &gl->gl_flags);
427
428 if (new_state != gl->gl_target)
429
430 gl->gl_hold_time = max(gl->gl_hold_time - GL_GLOCK_HOLD_DECR,
431 GL_GLOCK_MIN_HOLD);
432 gl->gl_state = new_state;
433 gl->gl_tchange = jiffies;
434}
435
436static void gfs2_demote_wake(struct gfs2_glock *gl)
437{
438 gl->gl_demote_state = LM_ST_EXCLUSIVE;
439 clear_bit(GLF_DEMOTE, &gl->gl_flags);
440 smp_mb__after_atomic();
441 wake_up_bit(&gl->gl_flags, GLF_DEMOTE);
442}
443
444
445
446
447
448
449
450
451static void finish_xmote(struct gfs2_glock *gl, unsigned int ret)
452{
453 const struct gfs2_glock_operations *glops = gl->gl_ops;
454 struct gfs2_holder *gh;
455 unsigned state = ret & LM_OUT_ST_MASK;
456 int rv;
457
458 spin_lock(&gl->gl_lockref.lock);
459 trace_gfs2_glock_state_change(gl, state);
460 state_change(gl, state);
461 gh = find_first_waiter(gl);
462
463
464 if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) &&
465 state != LM_ST_UNLOCKED && gl->gl_demote_state == LM_ST_UNLOCKED)
466 gl->gl_target = LM_ST_UNLOCKED;
467
468
469 if (unlikely(state != gl->gl_target)) {
470 if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) {
471
472 if (ret & LM_OUT_CANCELED) {
473 if ((gh->gh_flags & LM_FLAG_PRIORITY) == 0)
474 list_move_tail(&gh->gh_list, &gl->gl_holders);
475 gh = find_first_waiter(gl);
476 gl->gl_target = gh->gh_state;
477 goto retry;
478 }
479
480 if ((ret & LM_OUT_ERROR) ||
481 (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
482 gl->gl_target = gl->gl_state;
483 do_error(gl, ret);
484 goto out;
485 }
486 }
487 switch(state) {
488
489 case LM_ST_UNLOCKED:
490retry:
491 do_xmote(gl, gh, gl->gl_target);
492 break;
493
494 case LM_ST_SHARED:
495 case LM_ST_DEFERRED:
496 do_xmote(gl, gh, LM_ST_UNLOCKED);
497 break;
498 default:
499 fs_err(gl->gl_name.ln_sbd, "wanted %u got %u\n",
500 gl->gl_target, state);
501 GLOCK_BUG_ON(gl, 1);
502 }
503 spin_unlock(&gl->gl_lockref.lock);
504 return;
505 }
506
507
508 if (test_and_clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags))
509 gfs2_demote_wake(gl);
510 if (state != LM_ST_UNLOCKED) {
511 if (glops->go_xmote_bh) {
512 spin_unlock(&gl->gl_lockref.lock);
513 rv = glops->go_xmote_bh(gl, gh);
514 spin_lock(&gl->gl_lockref.lock);
515 if (rv) {
516 do_error(gl, rv);
517 goto out;
518 }
519 }
520 rv = do_promote(gl);
521 if (rv == 2)
522 goto out_locked;
523 }
524out:
525 clear_bit(GLF_LOCK, &gl->gl_flags);
526out_locked:
527 spin_unlock(&gl->gl_lockref.lock);
528}
529
530
531
532
533
534
535
536
537
538static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target)
539__releases(&gl->gl_lockref.lock)
540__acquires(&gl->gl_lockref.lock)
541{
542 const struct gfs2_glock_operations *glops = gl->gl_ops;
543 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
544 unsigned int lck_flags = (unsigned int)(gh ? gh->gh_flags : 0);
545 int ret;
546
547 if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags)) &&
548 target != LM_ST_UNLOCKED)
549 return;
550 lck_flags &= (LM_FLAG_TRY | LM_FLAG_TRY_1CB | LM_FLAG_NOEXP |
551 LM_FLAG_PRIORITY);
552 GLOCK_BUG_ON(gl, gl->gl_state == target);
553 GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target);
554 if ((target == LM_ST_UNLOCKED || target == LM_ST_DEFERRED) &&
555 glops->go_inval) {
556 set_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
557 do_error(gl, 0);
558 }
559 gl->gl_req = target;
560 set_bit(GLF_BLOCKING, &gl->gl_flags);
561 if ((gl->gl_req == LM_ST_UNLOCKED) ||
562 (gl->gl_state == LM_ST_EXCLUSIVE) ||
563 (lck_flags & (LM_FLAG_TRY|LM_FLAG_TRY_1CB)))
564 clear_bit(GLF_BLOCKING, &gl->gl_flags);
565 spin_unlock(&gl->gl_lockref.lock);
566 if (glops->go_sync)
567 glops->go_sync(gl);
568 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags))
569 glops->go_inval(gl, target == LM_ST_DEFERRED ? 0 : DIO_METADATA);
570 clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
571
572 gfs2_glock_hold(gl);
573 if (sdp->sd_lockstruct.ls_ops->lm_lock) {
574
575 ret = sdp->sd_lockstruct.ls_ops->lm_lock(gl, target, lck_flags);
576 if (ret == -EINVAL && gl->gl_target == LM_ST_UNLOCKED &&
577 target == LM_ST_UNLOCKED &&
578 test_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags)) {
579 finish_xmote(gl, target);
580 gfs2_glock_queue_work(gl, 0);
581 }
582 else if (ret) {
583 fs_err(sdp, "lm_lock ret %d\n", ret);
584 GLOCK_BUG_ON(gl, !test_bit(SDF_WITHDRAWN,
585 &sdp->sd_flags));
586 }
587 } else {
588 finish_xmote(gl, target);
589 gfs2_glock_queue_work(gl, 0);
590 }
591
592 spin_lock(&gl->gl_lockref.lock);
593}
594
595
596
597
598
599
600static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl)
601{
602 struct gfs2_holder *gh;
603
604 if (!list_empty(&gl->gl_holders)) {
605 gh = list_entry(gl->gl_holders.next, struct gfs2_holder, gh_list);
606 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
607 return gh;
608 }
609 return NULL;
610}
611
612
613
614
615
616
617
618
619static void run_queue(struct gfs2_glock *gl, const int nonblock)
620__releases(&gl->gl_lockref.lock)
621__acquires(&gl->gl_lockref.lock)
622{
623 struct gfs2_holder *gh = NULL;
624 int ret;
625
626 if (test_and_set_bit(GLF_LOCK, &gl->gl_flags))
627 return;
628
629 GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags));
630
631 if (test_bit(GLF_DEMOTE, &gl->gl_flags) &&
632 gl->gl_demote_state != gl->gl_state) {
633 if (find_first_holder(gl))
634 goto out_unlock;
635 if (nonblock)
636 goto out_sched;
637 set_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags);
638 GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE);
639 gl->gl_target = gl->gl_demote_state;
640 } else {
641 if (test_bit(GLF_DEMOTE, &gl->gl_flags))
642 gfs2_demote_wake(gl);
643 ret = do_promote(gl);
644 if (ret == 0)
645 goto out_unlock;
646 if (ret == 2)
647 goto out;
648 gh = find_first_waiter(gl);
649 gl->gl_target = gh->gh_state;
650 if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)))
651 do_error(gl, 0);
652 }
653 do_xmote(gl, gh, gl->gl_target);
654out:
655 return;
656
657out_sched:
658 clear_bit(GLF_LOCK, &gl->gl_flags);
659 smp_mb__after_atomic();
660 gl->gl_lockref.count++;
661 __gfs2_glock_queue_work(gl, 0);
662 return;
663
664out_unlock:
665 clear_bit(GLF_LOCK, &gl->gl_flags);
666 smp_mb__after_atomic();
667 return;
668}
669
670static void delete_work_func(struct work_struct *work)
671{
672 struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_delete);
673 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
674 struct inode *inode;
675 u64 no_addr = gl->gl_name.ln_number;
676
677
678
679
680 if (test_bit(GLF_INODE_CREATING, &gl->gl_flags))
681 goto out;
682
683 inode = gfs2_lookup_by_inum(sdp, no_addr, NULL, GFS2_BLKST_UNLINKED);
684 if (!IS_ERR_OR_NULL(inode)) {
685 d_prune_aliases(inode);
686 iput(inode);
687 }
688out:
689 gfs2_glock_put(gl);
690}
691
692static void glock_work_func(struct work_struct *work)
693{
694 unsigned long delay = 0;
695 struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work.work);
696 unsigned int drop_refs = 1;
697
698 if (test_and_clear_bit(GLF_REPLY_PENDING, &gl->gl_flags)) {
699 finish_xmote(gl, gl->gl_reply);
700 drop_refs++;
701 }
702 spin_lock(&gl->gl_lockref.lock);
703 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
704 gl->gl_state != LM_ST_UNLOCKED &&
705 gl->gl_demote_state != LM_ST_EXCLUSIVE) {
706 unsigned long holdtime, now = jiffies;
707
708 holdtime = gl->gl_tchange + gl->gl_hold_time;
709 if (time_before(now, holdtime))
710 delay = holdtime - now;
711
712 if (!delay) {
713 clear_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
714 set_bit(GLF_DEMOTE, &gl->gl_flags);
715 }
716 }
717 run_queue(gl, 0);
718 if (delay) {
719
720 drop_refs--;
721 if (gl->gl_name.ln_type != LM_TYPE_INODE)
722 delay = 0;
723 __gfs2_glock_queue_work(gl, delay);
724 }
725
726
727
728
729
730
731 gl->gl_lockref.count -= drop_refs;
732 if (!gl->gl_lockref.count) {
733 __gfs2_glock_put(gl);
734 return;
735 }
736 spin_unlock(&gl->gl_lockref.lock);
737}
738
739static struct gfs2_glock *find_insert_glock(struct lm_lockname *name,
740 struct gfs2_glock *new)
741{
742 struct wait_glock_queue wait;
743 wait_queue_head_t *wq = glock_waitqueue(name);
744 struct gfs2_glock *gl;
745
746 wait.name = name;
747 init_wait(&wait.wait);
748 wait.wait.func = glock_wake_function;
749
750again:
751 prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
752 rcu_read_lock();
753 if (new) {
754 gl = rhashtable_lookup_get_insert_fast(&gl_hash_table,
755 &new->gl_node, ht_parms);
756 if (IS_ERR(gl))
757 goto out;
758 } else {
759 gl = rhashtable_lookup_fast(&gl_hash_table,
760 name, ht_parms);
761 }
762 if (gl && !lockref_get_not_dead(&gl->gl_lockref)) {
763 rcu_read_unlock();
764 schedule();
765 goto again;
766 }
767out:
768 rcu_read_unlock();
769 finish_wait(wq, &wait.wait);
770 return gl;
771}
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
787 const struct gfs2_glock_operations *glops, int create,
788 struct gfs2_glock **glp)
789{
790 struct super_block *s = sdp->sd_vfs;
791 struct lm_lockname name = { .ln_number = number,
792 .ln_type = glops->go_type,
793 .ln_sbd = sdp };
794 struct gfs2_glock *gl, *tmp;
795 struct address_space *mapping;
796 struct kmem_cache *cachep;
797 int ret = 0;
798
799 gl = find_insert_glock(&name, NULL);
800 if (gl) {
801 *glp = gl;
802 return 0;
803 }
804 if (!create)
805 return -ENOENT;
806
807 if (glops->go_flags & GLOF_ASPACE)
808 cachep = gfs2_glock_aspace_cachep;
809 else
810 cachep = gfs2_glock_cachep;
811 gl = kmem_cache_alloc(cachep, GFP_NOFS);
812 if (!gl)
813 return -ENOMEM;
814
815 memset(&gl->gl_lksb, 0, sizeof(struct dlm_lksb));
816
817 if (glops->go_flags & GLOF_LVB) {
818 gl->gl_lksb.sb_lvbptr = kzalloc(GFS2_MIN_LVB_SIZE, GFP_NOFS);
819 if (!gl->gl_lksb.sb_lvbptr) {
820 kmem_cache_free(cachep, gl);
821 return -ENOMEM;
822 }
823 }
824
825 atomic_inc(&sdp->sd_glock_disposal);
826 gl->gl_node.next = NULL;
827 gl->gl_flags = 0;
828 gl->gl_name = name;
829 gl->gl_lockref.count = 1;
830 gl->gl_state = LM_ST_UNLOCKED;
831 gl->gl_target = LM_ST_UNLOCKED;
832 gl->gl_demote_state = LM_ST_EXCLUSIVE;
833 gl->gl_ops = glops;
834 gl->gl_dstamp = 0;
835 preempt_disable();
836
837 gl->gl_stats = this_cpu_ptr(sdp->sd_lkstats)->lkstats[glops->go_type];
838 preempt_enable();
839 gl->gl_stats.stats[GFS2_LKS_DCOUNT] = 0;
840 gl->gl_stats.stats[GFS2_LKS_QCOUNT] = 0;
841 gl->gl_tchange = jiffies;
842 gl->gl_object = NULL;
843 gl->gl_hold_time = GL_GLOCK_DFT_HOLD;
844 INIT_DELAYED_WORK(&gl->gl_work, glock_work_func);
845 INIT_WORK(&gl->gl_delete, delete_work_func);
846
847 mapping = gfs2_glock2aspace(gl);
848 if (mapping) {
849 mapping->a_ops = &gfs2_meta_aops;
850 mapping->host = s->s_bdev->bd_inode;
851 mapping->flags = 0;
852 mapping_set_gfp_mask(mapping, GFP_NOFS);
853 mapping->private_data = NULL;
854 mapping->writeback_index = 0;
855 }
856
857 tmp = find_insert_glock(&name, gl);
858 if (!tmp) {
859 *glp = gl;
860 goto out;
861 }
862 if (IS_ERR(tmp)) {
863 ret = PTR_ERR(tmp);
864 goto out_free;
865 }
866 *glp = tmp;
867
868out_free:
869 kfree(gl->gl_lksb.sb_lvbptr);
870 kmem_cache_free(cachep, gl);
871 atomic_dec(&sdp->sd_glock_disposal);
872
873out:
874 return ret;
875}
876
877
878
879
880
881
882
883
884
885
886void gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags,
887 struct gfs2_holder *gh)
888{
889 INIT_LIST_HEAD(&gh->gh_list);
890 gh->gh_gl = gl;
891 gh->gh_ip = _RET_IP_;
892 gh->gh_owner_pid = get_pid(task_pid(current));
893 gh->gh_state = state;
894 gh->gh_flags = flags;
895 gh->gh_error = 0;
896 gh->gh_iflags = 0;
897 gfs2_glock_hold(gl);
898}
899
900
901
902
903
904
905
906
907
908
909
910void gfs2_holder_reinit(unsigned int state, u16 flags, struct gfs2_holder *gh)
911{
912 gh->gh_state = state;
913 gh->gh_flags = flags;
914 gh->gh_iflags = 0;
915 gh->gh_ip = _RET_IP_;
916 put_pid(gh->gh_owner_pid);
917 gh->gh_owner_pid = get_pid(task_pid(current));
918}
919
920
921
922
923
924
925
926void gfs2_holder_uninit(struct gfs2_holder *gh)
927{
928 put_pid(gh->gh_owner_pid);
929 gfs2_glock_put(gh->gh_gl);
930 gfs2_holder_mark_uninitialized(gh);
931 gh->gh_ip = 0;
932}
933
934
935
936
937
938
939
940
941int gfs2_glock_wait(struct gfs2_holder *gh)
942{
943 unsigned long time1 = jiffies;
944
945 might_sleep();
946 wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);
947 if (time_after(jiffies, time1 + HZ))
948
949 gh->gh_gl->gl_hold_time = min(gh->gh_gl->gl_hold_time +
950 GL_GLOCK_HOLD_INCR,
951 GL_GLOCK_MAX_HOLD);
952 return gh->gh_error;
953}
954
955
956
957
958
959
960
961
962
963
964static void handle_callback(struct gfs2_glock *gl, unsigned int state,
965 unsigned long delay, bool remote)
966{
967 int bit = delay ? GLF_PENDING_DEMOTE : GLF_DEMOTE;
968
969 set_bit(bit, &gl->gl_flags);
970 if (gl->gl_demote_state == LM_ST_EXCLUSIVE) {
971 gl->gl_demote_state = state;
972 gl->gl_demote_time = jiffies;
973 } else if (gl->gl_demote_state != LM_ST_UNLOCKED &&
974 gl->gl_demote_state != state) {
975 gl->gl_demote_state = LM_ST_UNLOCKED;
976 }
977 if (gl->gl_ops->go_callback)
978 gl->gl_ops->go_callback(gl, remote);
979 trace_gfs2_demote_rq(gl, remote);
980}
981
982void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...)
983{
984 struct va_format vaf;
985 va_list args;
986
987 va_start(args, fmt);
988
989 if (seq) {
990 seq_vprintf(seq, fmt, args);
991 } else {
992 vaf.fmt = fmt;
993 vaf.va = &args;
994
995 pr_err("%pV", &vaf);
996 }
997
998 va_end(args);
999}
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011static inline void add_to_queue(struct gfs2_holder *gh)
1012__releases(&gl->gl_lockref.lock)
1013__acquires(&gl->gl_lockref.lock)
1014{
1015 struct gfs2_glock *gl = gh->gh_gl;
1016 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1017 struct list_head *insert_pt = NULL;
1018 struct gfs2_holder *gh2;
1019 int try_futile = 0;
1020
1021 BUG_ON(gh->gh_owner_pid == NULL);
1022 if (test_and_set_bit(HIF_WAIT, &gh->gh_iflags))
1023 BUG();
1024
1025 if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) {
1026 if (test_bit(GLF_LOCK, &gl->gl_flags))
1027 try_futile = !may_grant(gl, gh);
1028 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags))
1029 goto fail;
1030 }
1031
1032 list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
1033 if (unlikely(gh2->gh_owner_pid == gh->gh_owner_pid &&
1034 (gh->gh_gl->gl_ops->go_type != LM_TYPE_FLOCK)))
1035 goto trap_recursive;
1036 if (try_futile &&
1037 !(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
1038fail:
1039 gh->gh_error = GLR_TRYFAILED;
1040 gfs2_holder_wake(gh);
1041 return;
1042 }
1043 if (test_bit(HIF_HOLDER, &gh2->gh_iflags))
1044 continue;
1045 if (unlikely((gh->gh_flags & LM_FLAG_PRIORITY) && !insert_pt))
1046 insert_pt = &gh2->gh_list;
1047 }
1048 set_bit(GLF_QUEUED, &gl->gl_flags);
1049 trace_gfs2_glock_queue(gh, 1);
1050 gfs2_glstats_inc(gl, GFS2_LKS_QCOUNT);
1051 gfs2_sbstats_inc(gl, GFS2_LKS_QCOUNT);
1052 if (likely(insert_pt == NULL)) {
1053 list_add_tail(&gh->gh_list, &gl->gl_holders);
1054 if (unlikely(gh->gh_flags & LM_FLAG_PRIORITY))
1055 goto do_cancel;
1056 return;
1057 }
1058 list_add_tail(&gh->gh_list, insert_pt);
1059do_cancel:
1060 gh = list_entry(gl->gl_holders.next, struct gfs2_holder, gh_list);
1061 if (!(gh->gh_flags & LM_FLAG_PRIORITY)) {
1062 spin_unlock(&gl->gl_lockref.lock);
1063 if (sdp->sd_lockstruct.ls_ops->lm_cancel)
1064 sdp->sd_lockstruct.ls_ops->lm_cancel(gl);
1065 spin_lock(&gl->gl_lockref.lock);
1066 }
1067 return;
1068
1069trap_recursive:
1070 fs_err(sdp, "original: %pSR\n", (void *)gh2->gh_ip);
1071 fs_err(sdp, "pid: %d\n", pid_nr(gh2->gh_owner_pid));
1072 fs_err(sdp, "lock type: %d req lock state : %d\n",
1073 gh2->gh_gl->gl_name.ln_type, gh2->gh_state);
1074 fs_err(sdp, "new: %pSR\n", (void *)gh->gh_ip);
1075 fs_err(sdp, "pid: %d\n", pid_nr(gh->gh_owner_pid));
1076 fs_err(sdp, "lock type: %d req lock state : %d\n",
1077 gh->gh_gl->gl_name.ln_type, gh->gh_state);
1078 gfs2_dump_glock(NULL, gl, true);
1079 BUG();
1080}
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091int gfs2_glock_nq(struct gfs2_holder *gh)
1092{
1093 struct gfs2_glock *gl = gh->gh_gl;
1094 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1095 int error = 0;
1096
1097 if (unlikely(test_bit(SDF_WITHDRAWN, &sdp->sd_flags)))
1098 return -EIO;
1099
1100 if (test_bit(GLF_LRU, &gl->gl_flags))
1101 gfs2_glock_remove_from_lru(gl);
1102
1103 spin_lock(&gl->gl_lockref.lock);
1104 add_to_queue(gh);
1105 if (unlikely((LM_FLAG_NOEXP & gh->gh_flags) &&
1106 test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))) {
1107 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1108 gl->gl_lockref.count++;
1109 __gfs2_glock_queue_work(gl, 0);
1110 }
1111 run_queue(gl, 1);
1112 spin_unlock(&gl->gl_lockref.lock);
1113
1114 if (!(gh->gh_flags & GL_ASYNC))
1115 error = gfs2_glock_wait(gh);
1116
1117 return error;
1118}
1119
1120
1121
1122
1123
1124
1125
1126
1127int gfs2_glock_poll(struct gfs2_holder *gh)
1128{
1129 return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1;
1130}
1131
1132
1133
1134
1135
1136
1137
1138void gfs2_glock_dq(struct gfs2_holder *gh)
1139{
1140 struct gfs2_glock *gl = gh->gh_gl;
1141 const struct gfs2_glock_operations *glops = gl->gl_ops;
1142 unsigned delay = 0;
1143 int fast_path = 0;
1144
1145 spin_lock(&gl->gl_lockref.lock);
1146 if (gh->gh_flags & GL_NOCACHE)
1147 handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1148
1149 list_del_init(&gh->gh_list);
1150 clear_bit(HIF_HOLDER, &gh->gh_iflags);
1151 if (find_first_holder(gl) == NULL) {
1152 if (glops->go_unlock) {
1153 GLOCK_BUG_ON(gl, test_and_set_bit(GLF_LOCK, &gl->gl_flags));
1154 spin_unlock(&gl->gl_lockref.lock);
1155 glops->go_unlock(gh);
1156 spin_lock(&gl->gl_lockref.lock);
1157 clear_bit(GLF_LOCK, &gl->gl_flags);
1158 }
1159 if (list_empty(&gl->gl_holders) &&
1160 !test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1161 !test_bit(GLF_DEMOTE, &gl->gl_flags))
1162 fast_path = 1;
1163 }
1164 if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl))
1165 gfs2_glock_add_to_lru(gl);
1166
1167 trace_gfs2_glock_queue(gh, 0);
1168 if (unlikely(!fast_path)) {
1169 gl->gl_lockref.count++;
1170 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1171 !test_bit(GLF_DEMOTE, &gl->gl_flags) &&
1172 gl->gl_name.ln_type == LM_TYPE_INODE)
1173 delay = gl->gl_hold_time;
1174 __gfs2_glock_queue_work(gl, delay);
1175 }
1176 spin_unlock(&gl->gl_lockref.lock);
1177}
1178
1179void gfs2_glock_dq_wait(struct gfs2_holder *gh)
1180{
1181 struct gfs2_glock *gl = gh->gh_gl;
1182 gfs2_glock_dq(gh);
1183 might_sleep();
1184 wait_on_bit(&gl->gl_flags, GLF_DEMOTE, TASK_UNINTERRUPTIBLE);
1185}
1186
1187
1188
1189
1190
1191
1192
1193void gfs2_glock_dq_uninit(struct gfs2_holder *gh)
1194{
1195 gfs2_glock_dq(gh);
1196 gfs2_holder_uninit(gh);
1197}
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211int gfs2_glock_nq_num(struct gfs2_sbd *sdp, u64 number,
1212 const struct gfs2_glock_operations *glops,
1213 unsigned int state, u16 flags, struct gfs2_holder *gh)
1214{
1215 struct gfs2_glock *gl;
1216 int error;
1217
1218 error = gfs2_glock_get(sdp, number, glops, CREATE, &gl);
1219 if (!error) {
1220 error = gfs2_glock_nq_init(gl, state, flags, gh);
1221 gfs2_glock_put(gl);
1222 }
1223
1224 return error;
1225}
1226
1227
1228
1229
1230
1231
1232
1233
1234static int glock_compare(const void *arg_a, const void *arg_b)
1235{
1236 const struct gfs2_holder *gh_a = *(const struct gfs2_holder **)arg_a;
1237 const struct gfs2_holder *gh_b = *(const struct gfs2_holder **)arg_b;
1238 const struct lm_lockname *a = &gh_a->gh_gl->gl_name;
1239 const struct lm_lockname *b = &gh_b->gh_gl->gl_name;
1240
1241 if (a->ln_number > b->ln_number)
1242 return 1;
1243 if (a->ln_number < b->ln_number)
1244 return -1;
1245 BUG_ON(gh_a->gh_gl->gl_ops->go_type == gh_b->gh_gl->gl_ops->go_type);
1246 return 0;
1247}
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258static int nq_m_sync(unsigned int num_gh, struct gfs2_holder *ghs,
1259 struct gfs2_holder **p)
1260{
1261 unsigned int x;
1262 int error = 0;
1263
1264 for (x = 0; x < num_gh; x++)
1265 p[x] = &ghs[x];
1266
1267 sort(p, num_gh, sizeof(struct gfs2_holder *), glock_compare, NULL);
1268
1269 for (x = 0; x < num_gh; x++) {
1270 p[x]->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC);
1271
1272 error = gfs2_glock_nq(p[x]);
1273 if (error) {
1274 while (x--)
1275 gfs2_glock_dq(p[x]);
1276 break;
1277 }
1278 }
1279
1280 return error;
1281}
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293int gfs2_glock_nq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1294{
1295 struct gfs2_holder *tmp[4];
1296 struct gfs2_holder **pph = tmp;
1297 int error = 0;
1298
1299 switch(num_gh) {
1300 case 0:
1301 return 0;
1302 case 1:
1303 ghs->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC);
1304 return gfs2_glock_nq(ghs);
1305 default:
1306 if (num_gh <= 4)
1307 break;
1308 pph = kmalloc_array(num_gh, sizeof(struct gfs2_holder *),
1309 GFP_NOFS);
1310 if (!pph)
1311 return -ENOMEM;
1312 }
1313
1314 error = nq_m_sync(num_gh, ghs, pph);
1315
1316 if (pph != tmp)
1317 kfree(pph);
1318
1319 return error;
1320}
1321
1322
1323
1324
1325
1326
1327
1328
1329void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1330{
1331 while (num_gh--)
1332 gfs2_glock_dq(&ghs[num_gh]);
1333}
1334
1335void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state)
1336{
1337 unsigned long delay = 0;
1338 unsigned long holdtime;
1339 unsigned long now = jiffies;
1340
1341 gfs2_glock_hold(gl);
1342 holdtime = gl->gl_tchange + gl->gl_hold_time;
1343 if (test_bit(GLF_QUEUED, &gl->gl_flags) &&
1344 gl->gl_name.ln_type == LM_TYPE_INODE) {
1345 if (time_before(now, holdtime))
1346 delay = holdtime - now;
1347 if (test_bit(GLF_REPLY_PENDING, &gl->gl_flags))
1348 delay = gl->gl_hold_time;
1349 }
1350
1351 spin_lock(&gl->gl_lockref.lock);
1352 handle_callback(gl, state, delay, true);
1353 __gfs2_glock_queue_work(gl, delay);
1354 spin_unlock(&gl->gl_lockref.lock);
1355}
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368static int gfs2_should_freeze(const struct gfs2_glock *gl)
1369{
1370 const struct gfs2_holder *gh;
1371
1372 if (gl->gl_reply & ~LM_OUT_ST_MASK)
1373 return 0;
1374 if (gl->gl_target == LM_ST_UNLOCKED)
1375 return 0;
1376
1377 list_for_each_entry(gh, &gl->gl_holders, gh_list) {
1378 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
1379 continue;
1380 if (LM_FLAG_NOEXP & gh->gh_flags)
1381 return 0;
1382 }
1383
1384 return 1;
1385}
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396void gfs2_glock_complete(struct gfs2_glock *gl, int ret)
1397{
1398 struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct;
1399
1400 spin_lock(&gl->gl_lockref.lock);
1401 gl->gl_reply = ret;
1402
1403 if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags))) {
1404 if (gfs2_should_freeze(gl)) {
1405 set_bit(GLF_FROZEN, &gl->gl_flags);
1406 spin_unlock(&gl->gl_lockref.lock);
1407 return;
1408 }
1409 }
1410
1411 gl->gl_lockref.count++;
1412 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1413 __gfs2_glock_queue_work(gl, 0);
1414 spin_unlock(&gl->gl_lockref.lock);
1415}
1416
1417static int glock_cmp(void *priv, struct list_head *a, struct list_head *b)
1418{
1419 struct gfs2_glock *gla, *glb;
1420
1421 gla = list_entry(a, struct gfs2_glock, gl_lru);
1422 glb = list_entry(b, struct gfs2_glock, gl_lru);
1423
1424 if (gla->gl_name.ln_number > glb->gl_name.ln_number)
1425 return 1;
1426 if (gla->gl_name.ln_number < glb->gl_name.ln_number)
1427 return -1;
1428
1429 return 0;
1430}
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446static void gfs2_dispose_glock_lru(struct list_head *list)
1447__releases(&lru_lock)
1448__acquires(&lru_lock)
1449{
1450 struct gfs2_glock *gl;
1451
1452 list_sort(NULL, list, glock_cmp);
1453
1454 while(!list_empty(list)) {
1455 gl = list_entry(list->next, struct gfs2_glock, gl_lru);
1456 list_del_init(&gl->gl_lru);
1457 if (!spin_trylock(&gl->gl_lockref.lock)) {
1458add_back_to_lru:
1459 list_add(&gl->gl_lru, &lru_list);
1460 set_bit(GLF_LRU, &gl->gl_flags);
1461 atomic_inc(&lru_count);
1462 continue;
1463 }
1464 if (test_and_set_bit(GLF_LOCK, &gl->gl_flags)) {
1465 spin_unlock(&gl->gl_lockref.lock);
1466 goto add_back_to_lru;
1467 }
1468 gl->gl_lockref.count++;
1469 if (demote_ok(gl))
1470 handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1471 WARN_ON(!test_and_clear_bit(GLF_LOCK, &gl->gl_flags));
1472 __gfs2_glock_queue_work(gl, 0);
1473 spin_unlock(&gl->gl_lockref.lock);
1474 cond_resched_lock(&lru_lock);
1475 }
1476}
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487static long gfs2_scan_glock_lru(int nr)
1488{
1489 struct gfs2_glock *gl;
1490 LIST_HEAD(skipped);
1491 LIST_HEAD(dispose);
1492 long freed = 0;
1493
1494 spin_lock(&lru_lock);
1495 while ((nr-- >= 0) && !list_empty(&lru_list)) {
1496 gl = list_entry(lru_list.next, struct gfs2_glock, gl_lru);
1497
1498
1499 if (!test_bit(GLF_LOCK, &gl->gl_flags)) {
1500 list_move(&gl->gl_lru, &dispose);
1501 atomic_dec(&lru_count);
1502 clear_bit(GLF_LRU, &gl->gl_flags);
1503 freed++;
1504 continue;
1505 }
1506
1507 list_move(&gl->gl_lru, &skipped);
1508 }
1509 list_splice(&skipped, &lru_list);
1510 if (!list_empty(&dispose))
1511 gfs2_dispose_glock_lru(&dispose);
1512 spin_unlock(&lru_lock);
1513
1514 return freed;
1515}
1516
1517static unsigned long gfs2_glock_shrink_scan(struct shrinker *shrink,
1518 struct shrink_control *sc)
1519{
1520 if (!(sc->gfp_mask & __GFP_FS))
1521 return SHRINK_STOP;
1522 return gfs2_scan_glock_lru(sc->nr_to_scan);
1523}
1524
1525static unsigned long gfs2_glock_shrink_count(struct shrinker *shrink,
1526 struct shrink_control *sc)
1527{
1528 return vfs_pressure_ratio(atomic_read(&lru_count));
1529}
1530
1531static struct shrinker glock_shrinker = {
1532 .seeks = DEFAULT_SEEKS,
1533 .count_objects = gfs2_glock_shrink_count,
1534 .scan_objects = gfs2_glock_shrink_scan,
1535};
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp)
1549{
1550 struct gfs2_glock *gl;
1551 struct rhashtable_iter iter;
1552
1553 rhashtable_walk_enter(&gl_hash_table, &iter);
1554
1555 do {
1556 rhashtable_walk_start(&iter);
1557
1558 while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl))
1559 if (gl->gl_name.ln_sbd == sdp &&
1560 lockref_get_not_dead(&gl->gl_lockref))
1561 examiner(gl);
1562
1563 rhashtable_walk_stop(&iter);
1564 } while (cond_resched(), gl == ERR_PTR(-EAGAIN));
1565
1566 rhashtable_walk_exit(&iter);
1567}
1568
1569
1570
1571
1572
1573
1574
1575static void thaw_glock(struct gfs2_glock *gl)
1576{
1577 if (!test_and_clear_bit(GLF_FROZEN, &gl->gl_flags)) {
1578 gfs2_glock_put(gl);
1579 return;
1580 }
1581 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1582 gfs2_glock_queue_work(gl, 0);
1583}
1584
1585
1586
1587
1588
1589
1590
1591static void clear_glock(struct gfs2_glock *gl)
1592{
1593 gfs2_glock_remove_from_lru(gl);
1594
1595 spin_lock(&gl->gl_lockref.lock);
1596 if (gl->gl_state != LM_ST_UNLOCKED)
1597 handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1598 __gfs2_glock_queue_work(gl, 0);
1599 spin_unlock(&gl->gl_lockref.lock);
1600}
1601
1602
1603
1604
1605
1606
1607
1608void gfs2_glock_thaw(struct gfs2_sbd *sdp)
1609{
1610 glock_hash_walk(thaw_glock, sdp);
1611}
1612
1613static void dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
1614{
1615 spin_lock(&gl->gl_lockref.lock);
1616 gfs2_dump_glock(seq, gl, fsid);
1617 spin_unlock(&gl->gl_lockref.lock);
1618}
1619
1620static void dump_glock_func(struct gfs2_glock *gl)
1621{
1622 dump_glock(NULL, gl, true);
1623}
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633void gfs2_gl_hash_clear(struct gfs2_sbd *sdp)
1634{
1635 set_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags);
1636 flush_workqueue(glock_workqueue);
1637 glock_hash_walk(clear_glock, sdp);
1638 flush_workqueue(glock_workqueue);
1639 wait_event_timeout(sdp->sd_glock_wait,
1640 atomic_read(&sdp->sd_glock_disposal) == 0,
1641 HZ * 600);
1642 glock_hash_walk(dump_glock_func, sdp);
1643}
1644
1645void gfs2_glock_finish_truncate(struct gfs2_inode *ip)
1646{
1647 struct gfs2_glock *gl = ip->i_gl;
1648 int ret;
1649
1650 ret = gfs2_truncatei_resume(ip);
1651 gfs2_assert_withdraw(gl->gl_name.ln_sbd, ret == 0);
1652
1653 spin_lock(&gl->gl_lockref.lock);
1654 clear_bit(GLF_LOCK, &gl->gl_flags);
1655 run_queue(gl, 1);
1656 spin_unlock(&gl->gl_lockref.lock);
1657}
1658
1659static const char *state2str(unsigned state)
1660{
1661 switch(state) {
1662 case LM_ST_UNLOCKED:
1663 return "UN";
1664 case LM_ST_SHARED:
1665 return "SH";
1666 case LM_ST_DEFERRED:
1667 return "DF";
1668 case LM_ST_EXCLUSIVE:
1669 return "EX";
1670 }
1671 return "??";
1672}
1673
1674static const char *hflags2str(char *buf, u16 flags, unsigned long iflags)
1675{
1676 char *p = buf;
1677 if (flags & LM_FLAG_TRY)
1678 *p++ = 't';
1679 if (flags & LM_FLAG_TRY_1CB)
1680 *p++ = 'T';
1681 if (flags & LM_FLAG_NOEXP)
1682 *p++ = 'e';
1683 if (flags & LM_FLAG_ANY)
1684 *p++ = 'A';
1685 if (flags & LM_FLAG_PRIORITY)
1686 *p++ = 'p';
1687 if (flags & GL_ASYNC)
1688 *p++ = 'a';
1689 if (flags & GL_EXACT)
1690 *p++ = 'E';
1691 if (flags & GL_NOCACHE)
1692 *p++ = 'c';
1693 if (test_bit(HIF_HOLDER, &iflags))
1694 *p++ = 'H';
1695 if (test_bit(HIF_WAIT, &iflags))
1696 *p++ = 'W';
1697 if (test_bit(HIF_FIRST, &iflags))
1698 *p++ = 'F';
1699 *p = 0;
1700 return buf;
1701}
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh,
1712 const char *fs_id_buf)
1713{
1714 struct task_struct *gh_owner = NULL;
1715 char flags_buf[32];
1716
1717 rcu_read_lock();
1718 if (gh->gh_owner_pid)
1719 gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID);
1720 gfs2_print_dbg(seq, "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n",
1721 fs_id_buf, state2str(gh->gh_state),
1722 hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags),
1723 gh->gh_error,
1724 gh->gh_owner_pid ? (long)pid_nr(gh->gh_owner_pid) : -1,
1725 gh_owner ? gh_owner->comm : "(ended)",
1726 (void *)gh->gh_ip);
1727 rcu_read_unlock();
1728}
1729
1730static const char *gflags2str(char *buf, const struct gfs2_glock *gl)
1731{
1732 const unsigned long *gflags = &gl->gl_flags;
1733 char *p = buf;
1734
1735 if (test_bit(GLF_LOCK, gflags))
1736 *p++ = 'l';
1737 if (test_bit(GLF_DEMOTE, gflags))
1738 *p++ = 'D';
1739 if (test_bit(GLF_PENDING_DEMOTE, gflags))
1740 *p++ = 'd';
1741 if (test_bit(GLF_DEMOTE_IN_PROGRESS, gflags))
1742 *p++ = 'p';
1743 if (test_bit(GLF_DIRTY, gflags))
1744 *p++ = 'y';
1745 if (test_bit(GLF_LFLUSH, gflags))
1746 *p++ = 'f';
1747 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, gflags))
1748 *p++ = 'i';
1749 if (test_bit(GLF_REPLY_PENDING, gflags))
1750 *p++ = 'r';
1751 if (test_bit(GLF_INITIAL, gflags))
1752 *p++ = 'I';
1753 if (test_bit(GLF_FROZEN, gflags))
1754 *p++ = 'F';
1755 if (test_bit(GLF_QUEUED, gflags))
1756 *p++ = 'q';
1757 if (test_bit(GLF_LRU, gflags))
1758 *p++ = 'L';
1759 if (gl->gl_object)
1760 *p++ = 'o';
1761 if (test_bit(GLF_BLOCKING, gflags))
1762 *p++ = 'b';
1763 *p = 0;
1764 return buf;
1765}
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
1786{
1787 const struct gfs2_glock_operations *glops = gl->gl_ops;
1788 unsigned long long dtime;
1789 const struct gfs2_holder *gh;
1790 char gflags_buf[32];
1791 char fs_id_buf[GFS2_FSNAME_LEN + 3 * sizeof(int) + 2];
1792 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1793
1794 memset(fs_id_buf, 0, sizeof(fs_id_buf));
1795 if (fsid && sdp)
1796 sprintf(fs_id_buf, "fsid=%s: ", sdp->sd_fsname);
1797 dtime = jiffies - gl->gl_demote_time;
1798 dtime *= 1000000/HZ;
1799 if (!test_bit(GLF_DEMOTE, &gl->gl_flags))
1800 dtime = 0;
1801 gfs2_print_dbg(seq, "%sG: s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d "
1802 "v:%d r:%d m:%ld\n", fs_id_buf, state2str(gl->gl_state),
1803 gl->gl_name.ln_type,
1804 (unsigned long long)gl->gl_name.ln_number,
1805 gflags2str(gflags_buf, gl),
1806 state2str(gl->gl_target),
1807 state2str(gl->gl_demote_state), dtime,
1808 atomic_read(&gl->gl_ail_count),
1809 atomic_read(&gl->gl_revokes),
1810 (int)gl->gl_lockref.count, gl->gl_hold_time);
1811
1812 list_for_each_entry(gh, &gl->gl_holders, gh_list)
1813 dump_holder(seq, gh, fs_id_buf);
1814
1815 if (gl->gl_state != LM_ST_UNLOCKED && glops->go_dump)
1816 glops->go_dump(seq, gl, fs_id_buf);
1817}
1818
1819static int gfs2_glstats_seq_show(struct seq_file *seq, void *iter_ptr)
1820{
1821 struct gfs2_glock *gl = iter_ptr;
1822
1823 seq_printf(seq, "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n",
1824 gl->gl_name.ln_type,
1825 (unsigned long long)gl->gl_name.ln_number,
1826 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT],
1827 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR],
1828 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB],
1829 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVARB],
1830 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRT],
1831 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRTVAR],
1832 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_DCOUNT],
1833 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_QCOUNT]);
1834 return 0;
1835}
1836
1837static const char *gfs2_gltype[] = {
1838 "type",
1839 "reserved",
1840 "nondisk",
1841 "inode",
1842 "rgrp",
1843 "meta",
1844 "iopen",
1845 "flock",
1846 "plock",
1847 "quota",
1848 "journal",
1849};
1850
1851static const char *gfs2_stype[] = {
1852 [GFS2_LKS_SRTT] = "srtt",
1853 [GFS2_LKS_SRTTVAR] = "srttvar",
1854 [GFS2_LKS_SRTTB] = "srttb",
1855 [GFS2_LKS_SRTTVARB] = "srttvarb",
1856 [GFS2_LKS_SIRT] = "sirt",
1857 [GFS2_LKS_SIRTVAR] = "sirtvar",
1858 [GFS2_LKS_DCOUNT] = "dlm",
1859 [GFS2_LKS_QCOUNT] = "queue",
1860};
1861
1862#define GFS2_NR_SBSTATS (ARRAY_SIZE(gfs2_gltype) * ARRAY_SIZE(gfs2_stype))
1863
1864static int gfs2_sbstats_seq_show(struct seq_file *seq, void *iter_ptr)
1865{
1866 struct gfs2_sbd *sdp = seq->private;
1867 loff_t pos = *(loff_t *)iter_ptr;
1868 unsigned index = pos >> 3;
1869 unsigned subindex = pos & 0x07;
1870 int i;
1871
1872 if (index == 0 && subindex != 0)
1873 return 0;
1874
1875 seq_printf(seq, "%-10s %8s:", gfs2_gltype[index],
1876 (index == 0) ? "cpu": gfs2_stype[subindex]);
1877
1878 for_each_possible_cpu(i) {
1879 const struct gfs2_pcpu_lkstats *lkstats = per_cpu_ptr(sdp->sd_lkstats, i);
1880
1881 if (index == 0)
1882 seq_printf(seq, " %15u", i);
1883 else
1884 seq_printf(seq, " %15llu", (unsigned long long)lkstats->
1885 lkstats[index - 1].stats[subindex]);
1886 }
1887 seq_putc(seq, '\n');
1888 return 0;
1889}
1890
1891int __init gfs2_glock_init(void)
1892{
1893 int i, ret;
1894
1895 ret = rhashtable_init(&gl_hash_table, &ht_parms);
1896 if (ret < 0)
1897 return ret;
1898
1899 glock_workqueue = alloc_workqueue("glock_workqueue", WQ_MEM_RECLAIM |
1900 WQ_HIGHPRI | WQ_FREEZABLE, 0);
1901 if (!glock_workqueue) {
1902 rhashtable_destroy(&gl_hash_table);
1903 return -ENOMEM;
1904 }
1905 gfs2_delete_workqueue = alloc_workqueue("delete_workqueue",
1906 WQ_MEM_RECLAIM | WQ_FREEZABLE,
1907 0);
1908 if (!gfs2_delete_workqueue) {
1909 destroy_workqueue(glock_workqueue);
1910 rhashtable_destroy(&gl_hash_table);
1911 return -ENOMEM;
1912 }
1913
1914 ret = register_shrinker(&glock_shrinker);
1915 if (ret) {
1916 destroy_workqueue(gfs2_delete_workqueue);
1917 destroy_workqueue(glock_workqueue);
1918 rhashtable_destroy(&gl_hash_table);
1919 return ret;
1920 }
1921
1922 for (i = 0; i < GLOCK_WAIT_TABLE_SIZE; i++)
1923 init_waitqueue_head(glock_wait_table + i);
1924
1925 return 0;
1926}
1927
1928void gfs2_glock_exit(void)
1929{
1930 unregister_shrinker(&glock_shrinker);
1931 rhashtable_destroy(&gl_hash_table);
1932 destroy_workqueue(glock_workqueue);
1933 destroy_workqueue(gfs2_delete_workqueue);
1934}
1935
1936static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n)
1937{
1938 struct gfs2_glock *gl = gi->gl;
1939
1940 if (gl) {
1941 if (n == 0)
1942 return;
1943 if (!lockref_put_not_zero(&gl->gl_lockref))
1944 gfs2_glock_queue_put(gl);
1945 }
1946 for (;;) {
1947 gl = rhashtable_walk_next(&gi->hti);
1948 if (IS_ERR_OR_NULL(gl)) {
1949 if (gl == ERR_PTR(-EAGAIN)) {
1950 n = 1;
1951 continue;
1952 }
1953 gl = NULL;
1954 break;
1955 }
1956 if (gl->gl_name.ln_sbd != gi->sdp)
1957 continue;
1958 if (n <= 1) {
1959 if (!lockref_get_not_dead(&gl->gl_lockref))
1960 continue;
1961 break;
1962 } else {
1963 if (__lockref_is_dead(&gl->gl_lockref))
1964 continue;
1965 n--;
1966 }
1967 }
1968 gi->gl = gl;
1969}
1970
1971static void *gfs2_glock_seq_start(struct seq_file *seq, loff_t *pos)
1972 __acquires(RCU)
1973{
1974 struct gfs2_glock_iter *gi = seq->private;
1975 loff_t n;
1976
1977
1978
1979
1980
1981 if (*pos < gi->last_pos) {
1982 rhashtable_walk_exit(&gi->hti);
1983 rhashtable_walk_enter(&gl_hash_table, &gi->hti);
1984 n = *pos + 1;
1985 } else {
1986 n = *pos - gi->last_pos;
1987 }
1988
1989 rhashtable_walk_start(&gi->hti);
1990
1991 gfs2_glock_iter_next(gi, n);
1992 gi->last_pos = *pos;
1993 return gi->gl;
1994}
1995
1996static void *gfs2_glock_seq_next(struct seq_file *seq, void *iter_ptr,
1997 loff_t *pos)
1998{
1999 struct gfs2_glock_iter *gi = seq->private;
2000
2001 (*pos)++;
2002 gi->last_pos = *pos;
2003 gfs2_glock_iter_next(gi, 1);
2004 return gi->gl;
2005}
2006
2007static void gfs2_glock_seq_stop(struct seq_file *seq, void *iter_ptr)
2008 __releases(RCU)
2009{
2010 struct gfs2_glock_iter *gi = seq->private;
2011
2012 rhashtable_walk_stop(&gi->hti);
2013}
2014
2015static int gfs2_glock_seq_show(struct seq_file *seq, void *iter_ptr)
2016{
2017 dump_glock(seq, iter_ptr, false);
2018 return 0;
2019}
2020
2021static void *gfs2_sbstats_seq_start(struct seq_file *seq, loff_t *pos)
2022{
2023 preempt_disable();
2024 if (*pos >= GFS2_NR_SBSTATS)
2025 return NULL;
2026 return pos;
2027}
2028
2029static void *gfs2_sbstats_seq_next(struct seq_file *seq, void *iter_ptr,
2030 loff_t *pos)
2031{
2032 (*pos)++;
2033 if (*pos >= GFS2_NR_SBSTATS)
2034 return NULL;
2035 return pos;
2036}
2037
2038static void gfs2_sbstats_seq_stop(struct seq_file *seq, void *iter_ptr)
2039{
2040 preempt_enable();
2041}
2042
2043static const struct seq_operations gfs2_glock_seq_ops = {
2044 .start = gfs2_glock_seq_start,
2045 .next = gfs2_glock_seq_next,
2046 .stop = gfs2_glock_seq_stop,
2047 .show = gfs2_glock_seq_show,
2048};
2049
2050static const struct seq_operations gfs2_glstats_seq_ops = {
2051 .start = gfs2_glock_seq_start,
2052 .next = gfs2_glock_seq_next,
2053 .stop = gfs2_glock_seq_stop,
2054 .show = gfs2_glstats_seq_show,
2055};
2056
2057static const struct seq_operations gfs2_sbstats_seq_ops = {
2058 .start = gfs2_sbstats_seq_start,
2059 .next = gfs2_sbstats_seq_next,
2060 .stop = gfs2_sbstats_seq_stop,
2061 .show = gfs2_sbstats_seq_show,
2062};
2063
2064#define GFS2_SEQ_GOODSIZE min(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER, 65536UL)
2065
2066static int __gfs2_glocks_open(struct inode *inode, struct file *file,
2067 const struct seq_operations *ops)
2068{
2069 int ret = seq_open_private(file, ops, sizeof(struct gfs2_glock_iter));
2070 if (ret == 0) {
2071 struct seq_file *seq = file->private_data;
2072 struct gfs2_glock_iter *gi = seq->private;
2073
2074 gi->sdp = inode->i_private;
2075 seq->buf = kmalloc(GFS2_SEQ_GOODSIZE, GFP_KERNEL | __GFP_NOWARN);
2076 if (seq->buf)
2077 seq->size = GFS2_SEQ_GOODSIZE;
2078
2079
2080
2081
2082 gi->last_pos = -1;
2083 gi->gl = NULL;
2084 rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2085 }
2086 return ret;
2087}
2088
2089static int gfs2_glocks_open(struct inode *inode, struct file *file)
2090{
2091 return __gfs2_glocks_open(inode, file, &gfs2_glock_seq_ops);
2092}
2093
2094static int gfs2_glocks_release(struct inode *inode, struct file *file)
2095{
2096 struct seq_file *seq = file->private_data;
2097 struct gfs2_glock_iter *gi = seq->private;
2098
2099 if (gi->gl)
2100 gfs2_glock_put(gi->gl);
2101 rhashtable_walk_exit(&gi->hti);
2102 return seq_release_private(inode, file);
2103}
2104
2105static int gfs2_glstats_open(struct inode *inode, struct file *file)
2106{
2107 return __gfs2_glocks_open(inode, file, &gfs2_glstats_seq_ops);
2108}
2109
2110static int gfs2_sbstats_open(struct inode *inode, struct file *file)
2111{
2112 int ret = seq_open(file, &gfs2_sbstats_seq_ops);
2113 if (ret == 0) {
2114 struct seq_file *seq = file->private_data;
2115 seq->private = inode->i_private;
2116 }
2117 return ret;
2118}
2119
2120static const struct file_operations gfs2_glocks_fops = {
2121 .owner = THIS_MODULE,
2122 .open = gfs2_glocks_open,
2123 .read = seq_read,
2124 .llseek = seq_lseek,
2125 .release = gfs2_glocks_release,
2126};
2127
2128static const struct file_operations gfs2_glstats_fops = {
2129 .owner = THIS_MODULE,
2130 .open = gfs2_glstats_open,
2131 .read = seq_read,
2132 .llseek = seq_lseek,
2133 .release = gfs2_glocks_release,
2134};
2135
2136static const struct file_operations gfs2_sbstats_fops = {
2137 .owner = THIS_MODULE,
2138 .open = gfs2_sbstats_open,
2139 .read = seq_read,
2140 .llseek = seq_lseek,
2141 .release = seq_release,
2142};
2143
2144void gfs2_create_debugfs_file(struct gfs2_sbd *sdp)
2145{
2146 sdp->debugfs_dir = debugfs_create_dir(sdp->sd_table_name, gfs2_root);
2147
2148 debugfs_create_file("glocks", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2149 &gfs2_glocks_fops);
2150
2151 debugfs_create_file("glstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2152 &gfs2_glstats_fops);
2153
2154 debugfs_create_file("sbstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2155 &gfs2_sbstats_fops);
2156}
2157
2158void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp)
2159{
2160 debugfs_remove_recursive(sdp->debugfs_dir);
2161 sdp->debugfs_dir = NULL;
2162}
2163
2164void gfs2_register_debugfs(void)
2165{
2166 gfs2_root = debugfs_create_dir("gfs2", NULL);
2167}
2168
2169void gfs2_unregister_debugfs(void)
2170{
2171 debugfs_remove(gfs2_root);
2172 gfs2_root = NULL;
2173}
2174