1
2
3
4
5
6
7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9#include <linux/sched.h>
10#include <linux/slab.h>
11#include <linux/spinlock.h>
12#include <linux/buffer_head.h>
13#include <linux/delay.h>
14#include <linux/sort.h>
15#include <linux/hash.h>
16#include <linux/jhash.h>
17#include <linux/kallsyms.h>
18#include <linux/gfs2_ondisk.h>
19#include <linux/list.h>
20#include <linux/wait.h>
21#include <linux/module.h>
22#include <linux/uaccess.h>
23#include <linux/seq_file.h>
24#include <linux/debugfs.h>
25#include <linux/kthread.h>
26#include <linux/freezer.h>
27#include <linux/workqueue.h>
28#include <linux/jiffies.h>
29#include <linux/rcupdate.h>
30#include <linux/rculist_bl.h>
31#include <linux/bit_spinlock.h>
32#include <linux/percpu.h>
33#include <linux/list_sort.h>
34#include <linux/lockref.h>
35#include <linux/rhashtable.h>
36
37#include "gfs2.h"
38#include "incore.h"
39#include "glock.h"
40#include "glops.h"
41#include "inode.h"
42#include "lops.h"
43#include "meta_io.h"
44#include "quota.h"
45#include "super.h"
46#include "util.h"
47#include "bmap.h"
48#define CREATE_TRACE_POINTS
49#include "trace_gfs2.h"
50
51struct gfs2_glock_iter {
52 struct gfs2_sbd *sdp;
53 struct rhashtable_iter hti;
54 struct gfs2_glock *gl;
55 loff_t last_pos;
56};
57
58typedef void (*glock_examiner) (struct gfs2_glock * gl);
59
60static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
61static void __gfs2_glock_dq(struct gfs2_holder *gh);
62
63static struct dentry *gfs2_root;
64static struct workqueue_struct *glock_workqueue;
65struct workqueue_struct *gfs2_delete_workqueue;
66static LIST_HEAD(lru_list);
67static atomic_t lru_count = ATOMIC_INIT(0);
68static DEFINE_SPINLOCK(lru_lock);
69
70#define GFS2_GL_HASH_SHIFT 15
71#define GFS2_GL_HASH_SIZE BIT(GFS2_GL_HASH_SHIFT)
72
73static const struct rhashtable_params ht_parms = {
74 .nelem_hint = GFS2_GL_HASH_SIZE * 3 / 4,
75 .key_len = offsetofend(struct lm_lockname, ln_type),
76 .key_offset = offsetof(struct gfs2_glock, gl_name),
77 .head_offset = offsetof(struct gfs2_glock, gl_node),
78};
79
80static struct rhashtable gl_hash_table;
81
82#define GLOCK_WAIT_TABLE_BITS 12
83#define GLOCK_WAIT_TABLE_SIZE (1 << GLOCK_WAIT_TABLE_BITS)
84static wait_queue_head_t glock_wait_table[GLOCK_WAIT_TABLE_SIZE] __cacheline_aligned;
85
86struct wait_glock_queue {
87 struct lm_lockname *name;
88 wait_queue_entry_t wait;
89};
90
91static int glock_wake_function(wait_queue_entry_t *wait, unsigned int mode,
92 int sync, void *key)
93{
94 struct wait_glock_queue *wait_glock =
95 container_of(wait, struct wait_glock_queue, wait);
96 struct lm_lockname *wait_name = wait_glock->name;
97 struct lm_lockname *wake_name = key;
98
99 if (wake_name->ln_sbd != wait_name->ln_sbd ||
100 wake_name->ln_number != wait_name->ln_number ||
101 wake_name->ln_type != wait_name->ln_type)
102 return 0;
103 return autoremove_wake_function(wait, mode, sync, key);
104}
105
106static wait_queue_head_t *glock_waitqueue(struct lm_lockname *name)
107{
108 u32 hash = jhash2((u32 *)name, ht_parms.key_len / 4, 0);
109
110 return glock_wait_table + hash_32(hash, GLOCK_WAIT_TABLE_BITS);
111}
112
113
114
115
116
117static void wake_up_glock(struct gfs2_glock *gl)
118{
119 wait_queue_head_t *wq = glock_waitqueue(&gl->gl_name);
120
121 if (waitqueue_active(wq))
122 __wake_up(wq, TASK_NORMAL, 1, &gl->gl_name);
123}
124
125static void gfs2_glock_dealloc(struct rcu_head *rcu)
126{
127 struct gfs2_glock *gl = container_of(rcu, struct gfs2_glock, gl_rcu);
128
129 kfree(gl->gl_lksb.sb_lvbptr);
130 if (gl->gl_ops->go_flags & GLOF_ASPACE)
131 kmem_cache_free(gfs2_glock_aspace_cachep, gl);
132 else
133 kmem_cache_free(gfs2_glock_cachep, gl);
134}
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149static bool glock_blocked_by_withdraw(struct gfs2_glock *gl)
150{
151 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
152
153 if (likely(!gfs2_withdrawn(sdp)))
154 return false;
155 if (gl->gl_ops->go_flags & GLOF_NONDISK)
156 return false;
157 if (!sdp->sd_jdesc ||
158 gl->gl_name.ln_number == sdp->sd_jdesc->jd_no_addr)
159 return false;
160 return true;
161}
162
163void gfs2_glock_free(struct gfs2_glock *gl)
164{
165 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
166
167 gfs2_glock_assert_withdraw(gl, atomic_read(&gl->gl_revokes) == 0);
168 rhashtable_remove_fast(&gl_hash_table, &gl->gl_node, ht_parms);
169 smp_mb();
170 wake_up_glock(gl);
171 call_rcu(&gl->gl_rcu, gfs2_glock_dealloc);
172 if (atomic_dec_and_test(&sdp->sd_glock_disposal))
173 wake_up(&sdp->sd_glock_wait);
174}
175
176
177
178
179
180
181
182void gfs2_glock_hold(struct gfs2_glock *gl)
183{
184 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
185 lockref_get(&gl->gl_lockref);
186}
187
188
189
190
191
192
193
194
195static int demote_ok(const struct gfs2_glock *gl)
196{
197 const struct gfs2_glock_operations *glops = gl->gl_ops;
198
199 if (gl->gl_state == LM_ST_UNLOCKED)
200 return 0;
201
202
203
204
205
206
207 if (!list_empty(&gl->gl_holders))
208 return 0;
209 if (glops->go_demote_ok)
210 return glops->go_demote_ok(gl);
211 return 1;
212}
213
214
215void gfs2_glock_add_to_lru(struct gfs2_glock *gl)
216{
217 if (!(gl->gl_ops->go_flags & GLOF_LRU))
218 return;
219
220 spin_lock(&lru_lock);
221
222 list_move_tail(&gl->gl_lru, &lru_list);
223
224 if (!test_bit(GLF_LRU, &gl->gl_flags)) {
225 set_bit(GLF_LRU, &gl->gl_flags);
226 atomic_inc(&lru_count);
227 }
228
229 spin_unlock(&lru_lock);
230}
231
232static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl)
233{
234 if (!(gl->gl_ops->go_flags & GLOF_LRU))
235 return;
236
237 spin_lock(&lru_lock);
238 if (test_bit(GLF_LRU, &gl->gl_flags)) {
239 list_del_init(&gl->gl_lru);
240 atomic_dec(&lru_count);
241 clear_bit(GLF_LRU, &gl->gl_flags);
242 }
243 spin_unlock(&lru_lock);
244}
245
246
247
248
249
250static void __gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
251 if (!queue_delayed_work(glock_workqueue, &gl->gl_work, delay)) {
252
253
254
255
256
257
258 GLOCK_BUG_ON(gl, gl->gl_lockref.count < 2);
259 gl->gl_lockref.count--;
260 }
261}
262
263static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
264 spin_lock(&gl->gl_lockref.lock);
265 __gfs2_glock_queue_work(gl, delay);
266 spin_unlock(&gl->gl_lockref.lock);
267}
268
269static void __gfs2_glock_put(struct gfs2_glock *gl)
270{
271 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
272 struct address_space *mapping = gfs2_glock2aspace(gl);
273
274 lockref_mark_dead(&gl->gl_lockref);
275
276 gfs2_glock_remove_from_lru(gl);
277 spin_unlock(&gl->gl_lockref.lock);
278 GLOCK_BUG_ON(gl, !list_empty(&gl->gl_holders));
279 if (mapping) {
280 truncate_inode_pages_final(mapping);
281 if (!gfs2_withdrawn(sdp))
282 GLOCK_BUG_ON(gl, !mapping_empty(mapping));
283 }
284 trace_gfs2_glock_put(gl);
285 sdp->sd_lockstruct.ls_ops->lm_put_lock(gl);
286}
287
288
289
290
291void gfs2_glock_queue_put(struct gfs2_glock *gl)
292{
293 gfs2_glock_queue_work(gl, 0);
294}
295
296
297
298
299
300
301
302void gfs2_glock_put(struct gfs2_glock *gl)
303{
304 if (lockref_put_or_lock(&gl->gl_lockref))
305 return;
306
307 __gfs2_glock_put(gl);
308}
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324static inline bool may_grant(struct gfs2_glock *gl,
325 struct gfs2_holder *current_gh,
326 struct gfs2_holder *gh)
327{
328 if (current_gh) {
329 GLOCK_BUG_ON(gl, !test_bit(HIF_HOLDER, ¤t_gh->gh_iflags));
330
331 switch(current_gh->gh_state) {
332 case LM_ST_EXCLUSIVE:
333
334
335
336
337
338
339
340 return gh->gh_state == LM_ST_EXCLUSIVE &&
341 (current_gh->gh_flags & LM_FLAG_NODE_SCOPE) &&
342 (gh->gh_flags & LM_FLAG_NODE_SCOPE);
343
344 case LM_ST_SHARED:
345 case LM_ST_DEFERRED:
346 return gh->gh_state == current_gh->gh_state;
347
348 default:
349 return false;
350 }
351 }
352
353 if (gl->gl_state == gh->gh_state)
354 return true;
355 if (gh->gh_flags & GL_EXACT)
356 return false;
357 if (gl->gl_state == LM_ST_EXCLUSIVE) {
358 return gh->gh_state == LM_ST_SHARED ||
359 gh->gh_state == LM_ST_DEFERRED;
360 }
361 if (gh->gh_flags & LM_FLAG_ANY)
362 return gl->gl_state != LM_ST_UNLOCKED;
363 return false;
364}
365
366static void gfs2_holder_wake(struct gfs2_holder *gh)
367{
368 clear_bit(HIF_WAIT, &gh->gh_iflags);
369 smp_mb__after_atomic();
370 wake_up_bit(&gh->gh_iflags, HIF_WAIT);
371 if (gh->gh_flags & GL_ASYNC) {
372 struct gfs2_sbd *sdp = gh->gh_gl->gl_name.ln_sbd;
373
374 wake_up(&sdp->sd_async_glock_wait);
375 }
376}
377
378
379
380
381
382
383
384static void do_error(struct gfs2_glock *gl, const int ret)
385{
386 struct gfs2_holder *gh, *tmp;
387
388 list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
389 if (!test_bit(HIF_WAIT, &gh->gh_iflags))
390 continue;
391 if (ret & LM_OUT_ERROR)
392 gh->gh_error = -EIO;
393 else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))
394 gh->gh_error = GLR_TRYFAILED;
395 else
396 continue;
397 list_del_init(&gh->gh_list);
398 trace_gfs2_glock_queue(gh, 0);
399 gfs2_holder_wake(gh);
400 }
401}
402
403
404
405
406
407
408static void demote_incompat_holders(struct gfs2_glock *gl,
409 struct gfs2_holder *new_gh)
410{
411 struct gfs2_holder *gh, *tmp;
412
413
414
415
416
417
418 list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
419
420
421
422
423 if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
424 return;
425 if (test_bit(HIF_MAY_DEMOTE, &gh->gh_iflags) &&
426 !may_grant(gl, new_gh, gh)) {
427
428
429
430
431
432 __gfs2_glock_dq(gh);
433 }
434 }
435}
436
437
438
439
440
441
442static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl)
443{
444 struct gfs2_holder *gh;
445
446 if (!list_empty(&gl->gl_holders)) {
447 gh = list_first_entry(&gl->gl_holders, struct gfs2_holder,
448 gh_list);
449 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
450 return gh;
451 }
452 return NULL;
453}
454
455
456
457
458
459
460
461static inline struct gfs2_holder *
462find_first_strong_holder(struct gfs2_glock *gl)
463{
464 struct gfs2_holder *gh;
465
466 list_for_each_entry(gh, &gl->gl_holders, gh_list) {
467 if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
468 return NULL;
469 if (!test_bit(HIF_MAY_DEMOTE, &gh->gh_iflags))
470 return gh;
471 }
472 return NULL;
473}
474
475
476
477
478
479
480
481
482int gfs2_instantiate(struct gfs2_holder *gh)
483{
484 struct gfs2_glock *gl = gh->gh_gl;
485 const struct gfs2_glock_operations *glops = gl->gl_ops;
486 int ret;
487
488again:
489 if (!test_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags))
490 return 0;
491
492
493
494
495
496 if (test_and_set_bit(GLF_INSTANTIATE_IN_PROG, &gl->gl_flags)) {
497 wait_on_bit(&gl->gl_flags, GLF_INSTANTIATE_IN_PROG,
498 TASK_UNINTERRUPTIBLE);
499
500
501
502
503
504
505
506 goto again;
507 }
508
509 ret = glops->go_instantiate(gh);
510 if (!ret)
511 clear_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags);
512 clear_and_wake_up_bit(GLF_INSTANTIATE_IN_PROG, &gl->gl_flags);
513 return ret;
514}
515
516
517
518
519
520
521
522
523
524static int do_promote(struct gfs2_glock *gl)
525__releases(&gl->gl_lockref.lock)
526__acquires(&gl->gl_lockref.lock)
527{
528 struct gfs2_holder *gh, *tmp, *first_gh;
529 bool incompat_holders_demoted = false;
530 bool lock_released;
531 int ret;
532
533restart:
534 first_gh = find_first_strong_holder(gl);
535 list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
536 lock_released = false;
537 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
538 continue;
539 if (!may_grant(gl, first_gh, gh)) {
540
541
542
543
544
545 if (gh->gh_list.prev == &gl->gl_holders)
546 return 1;
547 do_error(gl, 0);
548 break;
549 }
550 if (!incompat_holders_demoted) {
551 demote_incompat_holders(gl, first_gh);
552 incompat_holders_demoted = true;
553 first_gh = gh;
554 }
555 if (test_bit(GLF_INSTANTIATE_NEEDED, &gl->gl_flags) &&
556 !(gh->gh_flags & GL_SKIP) && gl->gl_ops->go_instantiate) {
557 lock_released = true;
558 spin_unlock(&gl->gl_lockref.lock);
559 ret = gfs2_instantiate(gh);
560 spin_lock(&gl->gl_lockref.lock);
561 if (ret) {
562 if (ret == 1)
563 return 2;
564 gh->gh_error = ret;
565 list_del_init(&gh->gh_list);
566 trace_gfs2_glock_queue(gh, 0);
567 gfs2_holder_wake(gh);
568 goto restart;
569 }
570 }
571 set_bit(HIF_HOLDER, &gh->gh_iflags);
572 trace_gfs2_promote(gh);
573 gfs2_holder_wake(gh);
574
575
576
577
578
579 if (lock_released)
580 goto restart;
581 }
582 return 0;
583}
584
585
586
587
588
589
590static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl)
591{
592 struct gfs2_holder *gh;
593
594 list_for_each_entry(gh, &gl->gl_holders, gh_list) {
595 if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
596 return gh;
597 }
598 return NULL;
599}
600
601
602
603
604
605
606
607static void state_change(struct gfs2_glock *gl, unsigned int new_state)
608{
609 int held1, held2;
610
611 held1 = (gl->gl_state != LM_ST_UNLOCKED);
612 held2 = (new_state != LM_ST_UNLOCKED);
613
614 if (held1 != held2) {
615 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
616 if (held2)
617 gl->gl_lockref.count++;
618 else
619 gl->gl_lockref.count--;
620 }
621 if (new_state != gl->gl_target)
622
623 gl->gl_hold_time = max(gl->gl_hold_time - GL_GLOCK_HOLD_DECR,
624 GL_GLOCK_MIN_HOLD);
625 gl->gl_state = new_state;
626 gl->gl_tchange = jiffies;
627}
628
629static void gfs2_set_demote(struct gfs2_glock *gl)
630{
631 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
632
633 set_bit(GLF_DEMOTE, &gl->gl_flags);
634 smp_mb();
635 wake_up(&sdp->sd_async_glock_wait);
636}
637
638static void gfs2_demote_wake(struct gfs2_glock *gl)
639{
640 gl->gl_demote_state = LM_ST_EXCLUSIVE;
641 clear_bit(GLF_DEMOTE, &gl->gl_flags);
642 smp_mb__after_atomic();
643 wake_up_bit(&gl->gl_flags, GLF_DEMOTE);
644}
645
646
647
648
649
650
651
652
653static void finish_xmote(struct gfs2_glock *gl, unsigned int ret)
654{
655 const struct gfs2_glock_operations *glops = gl->gl_ops;
656 struct gfs2_holder *gh;
657 unsigned state = ret & LM_OUT_ST_MASK;
658 int rv;
659
660 spin_lock(&gl->gl_lockref.lock);
661 trace_gfs2_glock_state_change(gl, state);
662 state_change(gl, state);
663 gh = find_first_waiter(gl);
664
665
666 if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) &&
667 state != LM_ST_UNLOCKED && gl->gl_demote_state == LM_ST_UNLOCKED)
668 gl->gl_target = LM_ST_UNLOCKED;
669
670
671 if (unlikely(state != gl->gl_target)) {
672 if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) {
673
674 if (ret & LM_OUT_CANCELED) {
675 if ((gh->gh_flags & LM_FLAG_PRIORITY) == 0)
676 list_move_tail(&gh->gh_list, &gl->gl_holders);
677 gh = find_first_waiter(gl);
678 gl->gl_target = gh->gh_state;
679 goto retry;
680 }
681
682 if ((ret & LM_OUT_ERROR) ||
683 (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
684 gl->gl_target = gl->gl_state;
685 do_error(gl, ret);
686 goto out;
687 }
688 }
689 switch(state) {
690
691 case LM_ST_UNLOCKED:
692retry:
693 do_xmote(gl, gh, gl->gl_target);
694 break;
695
696 case LM_ST_SHARED:
697 case LM_ST_DEFERRED:
698 do_xmote(gl, gh, LM_ST_UNLOCKED);
699 break;
700 default:
701 fs_err(gl->gl_name.ln_sbd, "wanted %u got %u\n",
702 gl->gl_target, state);
703 GLOCK_BUG_ON(gl, 1);
704 }
705 spin_unlock(&gl->gl_lockref.lock);
706 return;
707 }
708
709
710 if (test_and_clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags))
711 gfs2_demote_wake(gl);
712 if (state != LM_ST_UNLOCKED) {
713 if (glops->go_xmote_bh) {
714 spin_unlock(&gl->gl_lockref.lock);
715 rv = glops->go_xmote_bh(gl);
716 spin_lock(&gl->gl_lockref.lock);
717 if (rv) {
718 do_error(gl, rv);
719 goto out;
720 }
721 }
722 rv = do_promote(gl);
723 if (rv == 2)
724 goto out_locked;
725 }
726out:
727 clear_bit(GLF_LOCK, &gl->gl_flags);
728out_locked:
729 spin_unlock(&gl->gl_lockref.lock);
730}
731
732static bool is_system_glock(struct gfs2_glock *gl)
733{
734 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
735 struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode);
736
737 if (gl == m_ip->i_gl)
738 return true;
739 return false;
740}
741
742
743
744
745
746
747
748
749
750static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target)
751__releases(&gl->gl_lockref.lock)
752__acquires(&gl->gl_lockref.lock)
753{
754 const struct gfs2_glock_operations *glops = gl->gl_ops;
755 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
756 unsigned int lck_flags = (unsigned int)(gh ? gh->gh_flags : 0);
757 int ret;
758
759 if (target != LM_ST_UNLOCKED && glock_blocked_by_withdraw(gl) &&
760 gh && !(gh->gh_flags & LM_FLAG_NOEXP))
761 return;
762 lck_flags &= (LM_FLAG_TRY | LM_FLAG_TRY_1CB | LM_FLAG_NOEXP |
763 LM_FLAG_PRIORITY);
764 GLOCK_BUG_ON(gl, gl->gl_state == target);
765 GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target);
766 if ((target == LM_ST_UNLOCKED || target == LM_ST_DEFERRED) &&
767 glops->go_inval) {
768
769
770
771
772
773 if (test_and_set_bit(GLF_INVALIDATE_IN_PROGRESS,
774 &gl->gl_flags))
775 return;
776 do_error(gl, 0);
777 }
778 gl->gl_req = target;
779 set_bit(GLF_BLOCKING, &gl->gl_flags);
780 if ((gl->gl_req == LM_ST_UNLOCKED) ||
781 (gl->gl_state == LM_ST_EXCLUSIVE) ||
782 (lck_flags & (LM_FLAG_TRY|LM_FLAG_TRY_1CB)))
783 clear_bit(GLF_BLOCKING, &gl->gl_flags);
784 spin_unlock(&gl->gl_lockref.lock);
785 if (glops->go_sync) {
786 ret = glops->go_sync(gl);
787
788
789
790
791 if (ret) {
792 if (cmpxchg(&sdp->sd_log_error, 0, ret)) {
793 fs_err(sdp, "Error %d syncing glock \n", ret);
794 gfs2_dump_glock(NULL, gl, true);
795 }
796 goto skip_inval;
797 }
798 }
799 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags)) {
800
801
802
803
804
805
806
807 if ((atomic_read(&gl->gl_ail_count) != 0) &&
808 (!cmpxchg(&sdp->sd_log_error, 0, -EIO))) {
809 gfs2_glock_assert_warn(gl,
810 !atomic_read(&gl->gl_ail_count));
811 gfs2_dump_glock(NULL, gl, true);
812 }
813 glops->go_inval(gl, target == LM_ST_DEFERRED ? 0 : DIO_METADATA);
814 clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
815 }
816
817skip_inval:
818 gfs2_glock_hold(gl);
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840 if (unlikely(sdp->sd_log_error && !gfs2_withdrawn(sdp)))
841 gfs2_withdraw_delayed(sdp);
842 if (glock_blocked_by_withdraw(gl) &&
843 (target != LM_ST_UNLOCKED ||
844 test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags))) {
845 if (!is_system_glock(gl)) {
846 gfs2_glock_queue_work(gl, GL_GLOCK_DFT_HOLD);
847 goto out;
848 } else {
849 clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
850 }
851 }
852
853 if (sdp->sd_lockstruct.ls_ops->lm_lock) {
854
855 ret = sdp->sd_lockstruct.ls_ops->lm_lock(gl, target, lck_flags);
856 if (ret == -EINVAL && gl->gl_target == LM_ST_UNLOCKED &&
857 target == LM_ST_UNLOCKED &&
858 test_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags)) {
859 finish_xmote(gl, target);
860 gfs2_glock_queue_work(gl, 0);
861 } else if (ret) {
862 fs_err(sdp, "lm_lock ret %d\n", ret);
863 GLOCK_BUG_ON(gl, !gfs2_withdrawn(sdp));
864 }
865 } else {
866 finish_xmote(gl, target);
867 gfs2_glock_queue_work(gl, 0);
868 }
869out:
870 spin_lock(&gl->gl_lockref.lock);
871}
872
873
874
875
876
877
878
879
880static void run_queue(struct gfs2_glock *gl, const int nonblock)
881__releases(&gl->gl_lockref.lock)
882__acquires(&gl->gl_lockref.lock)
883{
884 struct gfs2_holder *gh = NULL;
885 int ret;
886
887 if (test_and_set_bit(GLF_LOCK, &gl->gl_flags))
888 return;
889
890 GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags));
891
892 if (test_bit(GLF_DEMOTE, &gl->gl_flags) &&
893 gl->gl_demote_state != gl->gl_state) {
894 if (find_first_holder(gl))
895 goto out_unlock;
896 if (nonblock)
897 goto out_sched;
898 set_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags);
899 GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE);
900 gl->gl_target = gl->gl_demote_state;
901 } else {
902 if (test_bit(GLF_DEMOTE, &gl->gl_flags))
903 gfs2_demote_wake(gl);
904 ret = do_promote(gl);
905 if (ret == 0)
906 goto out_unlock;
907 if (ret == 2)
908 goto out;
909 gh = find_first_waiter(gl);
910 gl->gl_target = gh->gh_state;
911 if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)))
912 do_error(gl, 0);
913 }
914 do_xmote(gl, gh, gl->gl_target);
915out:
916 return;
917
918out_sched:
919 clear_bit(GLF_LOCK, &gl->gl_flags);
920 smp_mb__after_atomic();
921 gl->gl_lockref.count++;
922 __gfs2_glock_queue_work(gl, 0);
923 return;
924
925out_unlock:
926 clear_bit(GLF_LOCK, &gl->gl_flags);
927 smp_mb__after_atomic();
928 return;
929}
930
931void gfs2_inode_remember_delete(struct gfs2_glock *gl, u64 generation)
932{
933 struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
934
935 if (ri->ri_magic == 0)
936 ri->ri_magic = cpu_to_be32(GFS2_MAGIC);
937 if (ri->ri_magic == cpu_to_be32(GFS2_MAGIC))
938 ri->ri_generation_deleted = cpu_to_be64(generation);
939}
940
941bool gfs2_inode_already_deleted(struct gfs2_glock *gl, u64 generation)
942{
943 struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
944
945 if (ri->ri_magic != cpu_to_be32(GFS2_MAGIC))
946 return false;
947 return generation <= be64_to_cpu(ri->ri_generation_deleted);
948}
949
950static void gfs2_glock_poke(struct gfs2_glock *gl)
951{
952 int flags = LM_FLAG_TRY_1CB | LM_FLAG_ANY | GL_SKIP;
953 struct gfs2_holder gh;
954 int error;
955
956 __gfs2_holder_init(gl, LM_ST_SHARED, flags, &gh, _RET_IP_);
957 error = gfs2_glock_nq(&gh);
958 if (!error)
959 gfs2_glock_dq(&gh);
960 gfs2_holder_uninit(&gh);
961}
962
963static bool gfs2_try_evict(struct gfs2_glock *gl)
964{
965 struct gfs2_inode *ip;
966 bool evicted = false;
967
968
969
970
971
972
973
974
975
976
977
978 spin_lock(&gl->gl_lockref.lock);
979 ip = gl->gl_object;
980 if (ip && !igrab(&ip->i_inode))
981 ip = NULL;
982 spin_unlock(&gl->gl_lockref.lock);
983 if (ip) {
984 struct gfs2_glock *inode_gl = NULL;
985
986 gl->gl_no_formal_ino = ip->i_no_formal_ino;
987 set_bit(GIF_DEFERRED_DELETE, &ip->i_flags);
988 d_prune_aliases(&ip->i_inode);
989 iput(&ip->i_inode);
990
991
992 spin_lock(&gl->gl_lockref.lock);
993 ip = gl->gl_object;
994 if (ip) {
995 inode_gl = ip->i_gl;
996 lockref_get(&inode_gl->gl_lockref);
997 clear_bit(GIF_DEFERRED_DELETE, &ip->i_flags);
998 }
999 spin_unlock(&gl->gl_lockref.lock);
1000 if (inode_gl) {
1001 gfs2_glock_poke(inode_gl);
1002 gfs2_glock_put(inode_gl);
1003 }
1004 evicted = !ip;
1005 }
1006 return evicted;
1007}
1008
1009static void delete_work_func(struct work_struct *work)
1010{
1011 struct delayed_work *dwork = to_delayed_work(work);
1012 struct gfs2_glock *gl = container_of(dwork, struct gfs2_glock, gl_delete);
1013 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1014 struct inode *inode;
1015 u64 no_addr = gl->gl_name.ln_number;
1016
1017 spin_lock(&gl->gl_lockref.lock);
1018 clear_bit(GLF_PENDING_DELETE, &gl->gl_flags);
1019 spin_unlock(&gl->gl_lockref.lock);
1020
1021 if (test_bit(GLF_DEMOTE, &gl->gl_flags)) {
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039 if (gfs2_try_evict(gl)) {
1040 if (gfs2_queue_delete_work(gl, 5 * HZ))
1041 return;
1042 }
1043 goto out;
1044 }
1045
1046 inode = gfs2_lookup_by_inum(sdp, no_addr, gl->gl_no_formal_ino,
1047 GFS2_BLKST_UNLINKED);
1048 if (!IS_ERR_OR_NULL(inode)) {
1049 d_prune_aliases(inode);
1050 iput(inode);
1051 }
1052out:
1053 gfs2_glock_put(gl);
1054}
1055
1056static void glock_work_func(struct work_struct *work)
1057{
1058 unsigned long delay = 0;
1059 struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work.work);
1060 unsigned int drop_refs = 1;
1061
1062 if (test_and_clear_bit(GLF_REPLY_PENDING, &gl->gl_flags)) {
1063 finish_xmote(gl, gl->gl_reply);
1064 drop_refs++;
1065 }
1066 spin_lock(&gl->gl_lockref.lock);
1067 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1068 gl->gl_state != LM_ST_UNLOCKED &&
1069 gl->gl_demote_state != LM_ST_EXCLUSIVE) {
1070 unsigned long holdtime, now = jiffies;
1071
1072 holdtime = gl->gl_tchange + gl->gl_hold_time;
1073 if (time_before(now, holdtime))
1074 delay = holdtime - now;
1075
1076 if (!delay) {
1077 clear_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
1078 gfs2_set_demote(gl);
1079 }
1080 }
1081 run_queue(gl, 0);
1082 if (delay) {
1083
1084 drop_refs--;
1085 if (gl->gl_name.ln_type != LM_TYPE_INODE)
1086 delay = 0;
1087 __gfs2_glock_queue_work(gl, delay);
1088 }
1089
1090
1091
1092
1093
1094
1095 gl->gl_lockref.count -= drop_refs;
1096 if (!gl->gl_lockref.count) {
1097 __gfs2_glock_put(gl);
1098 return;
1099 }
1100 spin_unlock(&gl->gl_lockref.lock);
1101}
1102
1103static struct gfs2_glock *find_insert_glock(struct lm_lockname *name,
1104 struct gfs2_glock *new)
1105{
1106 struct wait_glock_queue wait;
1107 wait_queue_head_t *wq = glock_waitqueue(name);
1108 struct gfs2_glock *gl;
1109
1110 wait.name = name;
1111 init_wait(&wait.wait);
1112 wait.wait.func = glock_wake_function;
1113
1114again:
1115 prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
1116 rcu_read_lock();
1117 if (new) {
1118 gl = rhashtable_lookup_get_insert_fast(&gl_hash_table,
1119 &new->gl_node, ht_parms);
1120 if (IS_ERR(gl))
1121 goto out;
1122 } else {
1123 gl = rhashtable_lookup_fast(&gl_hash_table,
1124 name, ht_parms);
1125 }
1126 if (gl && !lockref_get_not_dead(&gl->gl_lockref)) {
1127 rcu_read_unlock();
1128 schedule();
1129 goto again;
1130 }
1131out:
1132 rcu_read_unlock();
1133 finish_wait(wq, &wait.wait);
1134 return gl;
1135}
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
1151 const struct gfs2_glock_operations *glops, int create,
1152 struct gfs2_glock **glp)
1153{
1154 struct super_block *s = sdp->sd_vfs;
1155 struct lm_lockname name = { .ln_number = number,
1156 .ln_type = glops->go_type,
1157 .ln_sbd = sdp };
1158 struct gfs2_glock *gl, *tmp;
1159 struct address_space *mapping;
1160 struct kmem_cache *cachep;
1161 int ret = 0;
1162
1163 gl = find_insert_glock(&name, NULL);
1164 if (gl) {
1165 *glp = gl;
1166 return 0;
1167 }
1168 if (!create)
1169 return -ENOENT;
1170
1171 if (glops->go_flags & GLOF_ASPACE)
1172 cachep = gfs2_glock_aspace_cachep;
1173 else
1174 cachep = gfs2_glock_cachep;
1175 gl = kmem_cache_alloc(cachep, GFP_NOFS);
1176 if (!gl)
1177 return -ENOMEM;
1178
1179 memset(&gl->gl_lksb, 0, sizeof(struct dlm_lksb));
1180
1181 if (glops->go_flags & GLOF_LVB) {
1182 gl->gl_lksb.sb_lvbptr = kzalloc(GDLM_LVB_SIZE, GFP_NOFS);
1183 if (!gl->gl_lksb.sb_lvbptr) {
1184 kmem_cache_free(cachep, gl);
1185 return -ENOMEM;
1186 }
1187 }
1188
1189 atomic_inc(&sdp->sd_glock_disposal);
1190 gl->gl_node.next = NULL;
1191 gl->gl_flags = glops->go_instantiate ? BIT(GLF_INSTANTIATE_NEEDED) : 0;
1192 gl->gl_name = name;
1193 lockdep_set_subclass(&gl->gl_lockref.lock, glops->go_subclass);
1194 gl->gl_lockref.count = 1;
1195 gl->gl_state = LM_ST_UNLOCKED;
1196 gl->gl_target = LM_ST_UNLOCKED;
1197 gl->gl_demote_state = LM_ST_EXCLUSIVE;
1198 gl->gl_ops = glops;
1199 gl->gl_dstamp = 0;
1200 preempt_disable();
1201
1202 gl->gl_stats = this_cpu_ptr(sdp->sd_lkstats)->lkstats[glops->go_type];
1203 preempt_enable();
1204 gl->gl_stats.stats[GFS2_LKS_DCOUNT] = 0;
1205 gl->gl_stats.stats[GFS2_LKS_QCOUNT] = 0;
1206 gl->gl_tchange = jiffies;
1207 gl->gl_object = NULL;
1208 gl->gl_hold_time = GL_GLOCK_DFT_HOLD;
1209 INIT_DELAYED_WORK(&gl->gl_work, glock_work_func);
1210 if (gl->gl_name.ln_type == LM_TYPE_IOPEN)
1211 INIT_DELAYED_WORK(&gl->gl_delete, delete_work_func);
1212
1213 mapping = gfs2_glock2aspace(gl);
1214 if (mapping) {
1215 mapping->a_ops = &gfs2_meta_aops;
1216 mapping->host = s->s_bdev->bd_inode;
1217 mapping->flags = 0;
1218 mapping_set_gfp_mask(mapping, GFP_NOFS);
1219 mapping->private_data = NULL;
1220 mapping->writeback_index = 0;
1221 }
1222
1223 tmp = find_insert_glock(&name, gl);
1224 if (!tmp) {
1225 *glp = gl;
1226 goto out;
1227 }
1228 if (IS_ERR(tmp)) {
1229 ret = PTR_ERR(tmp);
1230 goto out_free;
1231 }
1232 *glp = tmp;
1233
1234out_free:
1235 kfree(gl->gl_lksb.sb_lvbptr);
1236 kmem_cache_free(cachep, gl);
1237 if (atomic_dec_and_test(&sdp->sd_glock_disposal))
1238 wake_up(&sdp->sd_glock_wait);
1239
1240out:
1241 return ret;
1242}
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253void __gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags,
1254 struct gfs2_holder *gh, unsigned long ip)
1255{
1256 INIT_LIST_HEAD(&gh->gh_list);
1257 gh->gh_gl = gl;
1258 gh->gh_ip = ip;
1259 gh->gh_owner_pid = get_pid(task_pid(current));
1260 gh->gh_state = state;
1261 gh->gh_flags = flags;
1262 gh->gh_error = 0;
1263 gh->gh_iflags = 0;
1264 gfs2_glock_hold(gl);
1265}
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277void gfs2_holder_reinit(unsigned int state, u16 flags, struct gfs2_holder *gh)
1278{
1279 gh->gh_state = state;
1280 gh->gh_flags = flags;
1281 gh->gh_iflags = 0;
1282 gh->gh_ip = _RET_IP_;
1283 put_pid(gh->gh_owner_pid);
1284 gh->gh_owner_pid = get_pid(task_pid(current));
1285}
1286
1287
1288
1289
1290
1291
1292
1293void gfs2_holder_uninit(struct gfs2_holder *gh)
1294{
1295 put_pid(gh->gh_owner_pid);
1296 gfs2_glock_put(gh->gh_gl);
1297 gfs2_holder_mark_uninitialized(gh);
1298 gh->gh_ip = 0;
1299}
1300
1301static void gfs2_glock_update_hold_time(struct gfs2_glock *gl,
1302 unsigned long start_time)
1303{
1304
1305 if (time_after(jiffies, start_time + HZ)) {
1306
1307 gl->gl_hold_time = min(gl->gl_hold_time + GL_GLOCK_HOLD_INCR,
1308 GL_GLOCK_MAX_HOLD);
1309 }
1310}
1311
1312
1313
1314
1315
1316
1317
1318
1319int gfs2_glock_wait(struct gfs2_holder *gh)
1320{
1321 unsigned long start_time = jiffies;
1322
1323 might_sleep();
1324 wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);
1325 gfs2_glock_update_hold_time(gh->gh_gl, start_time);
1326 return gh->gh_error;
1327}
1328
1329static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs)
1330{
1331 int i;
1332
1333 for (i = 0; i < num_gh; i++)
1334 if (test_bit(HIF_WAIT, &ghs[i].gh_iflags))
1335 return 1;
1336 return 0;
1337}
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs)
1350{
1351 struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd;
1352 int i, ret = 0, timeout = 0;
1353 unsigned long start_time = jiffies;
1354 bool keep_waiting;
1355
1356 might_sleep();
1357
1358
1359
1360
1361 for (i = 0; i < num_gh; i++)
1362 timeout += ghs[i].gh_gl->gl_hold_time << 1;
1363
1364wait_for_dlm:
1365 if (!wait_event_timeout(sdp->sd_async_glock_wait,
1366 !glocks_pending(num_gh, ghs), timeout))
1367 ret = -ESTALE;
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383 keep_waiting = false;
1384 for (i = 0; i < num_gh; i++) {
1385
1386 if (!gfs2_holder_queued(&ghs[i]))
1387 continue;
1388
1389 if (test_bit(HIF_WAIT, &ghs[i].gh_iflags)) {
1390 keep_waiting = true;
1391 continue;
1392 }
1393
1394 if (test_bit(HIF_HOLDER, &ghs[i].gh_iflags)) {
1395 if (ret == -ESTALE)
1396 gfs2_glock_dq(&ghs[i]);
1397 else
1398 gfs2_glock_update_hold_time(ghs[i].gh_gl,
1399 start_time);
1400 }
1401 if (!ret)
1402 ret = ghs[i].gh_error;
1403 }
1404
1405 if (keep_waiting)
1406 goto wait_for_dlm;
1407
1408
1409
1410
1411 return ret;
1412}
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425static void handle_callback(struct gfs2_glock *gl, unsigned int state,
1426 unsigned long delay, bool remote)
1427{
1428 if (delay)
1429 set_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
1430 else
1431 gfs2_set_demote(gl);
1432 if (gl->gl_demote_state == LM_ST_EXCLUSIVE) {
1433 gl->gl_demote_state = state;
1434 gl->gl_demote_time = jiffies;
1435 } else if (gl->gl_demote_state != LM_ST_UNLOCKED &&
1436 gl->gl_demote_state != state) {
1437 gl->gl_demote_state = LM_ST_UNLOCKED;
1438 }
1439 if (gl->gl_ops->go_callback)
1440 gl->gl_ops->go_callback(gl, remote);
1441 trace_gfs2_demote_rq(gl, remote);
1442}
1443
1444void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...)
1445{
1446 struct va_format vaf;
1447 va_list args;
1448
1449 va_start(args, fmt);
1450
1451 if (seq) {
1452 seq_vprintf(seq, fmt, args);
1453 } else {
1454 vaf.fmt = fmt;
1455 vaf.va = &args;
1456
1457 pr_err("%pV", &vaf);
1458 }
1459
1460 va_end(args);
1461}
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473static inline void add_to_queue(struct gfs2_holder *gh)
1474__releases(&gl->gl_lockref.lock)
1475__acquires(&gl->gl_lockref.lock)
1476{
1477 struct gfs2_glock *gl = gh->gh_gl;
1478 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1479 struct list_head *insert_pt = NULL;
1480 struct gfs2_holder *gh2;
1481 int try_futile = 0;
1482
1483 GLOCK_BUG_ON(gl, gh->gh_owner_pid == NULL);
1484 if (test_and_set_bit(HIF_WAIT, &gh->gh_iflags))
1485 GLOCK_BUG_ON(gl, true);
1486
1487 if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) {
1488 if (test_bit(GLF_LOCK, &gl->gl_flags)) {
1489 struct gfs2_holder *first_gh;
1490
1491 first_gh = find_first_strong_holder(gl);
1492 try_futile = !may_grant(gl, first_gh, gh);
1493 }
1494 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags))
1495 goto fail;
1496 }
1497
1498 list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
1499 if (unlikely(gh2->gh_owner_pid == gh->gh_owner_pid &&
1500 (gh->gh_gl->gl_ops->go_type != LM_TYPE_FLOCK) &&
1501 !test_bit(HIF_MAY_DEMOTE, &gh2->gh_iflags)))
1502 goto trap_recursive;
1503 if (try_futile &&
1504 !(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
1505fail:
1506 gh->gh_error = GLR_TRYFAILED;
1507 gfs2_holder_wake(gh);
1508 return;
1509 }
1510 if (test_bit(HIF_HOLDER, &gh2->gh_iflags))
1511 continue;
1512 if (unlikely((gh->gh_flags & LM_FLAG_PRIORITY) && !insert_pt))
1513 insert_pt = &gh2->gh_list;
1514 }
1515 trace_gfs2_glock_queue(gh, 1);
1516 gfs2_glstats_inc(gl, GFS2_LKS_QCOUNT);
1517 gfs2_sbstats_inc(gl, GFS2_LKS_QCOUNT);
1518 if (likely(insert_pt == NULL)) {
1519 list_add_tail(&gh->gh_list, &gl->gl_holders);
1520 if (unlikely(gh->gh_flags & LM_FLAG_PRIORITY))
1521 goto do_cancel;
1522 return;
1523 }
1524 list_add_tail(&gh->gh_list, insert_pt);
1525do_cancel:
1526 gh = list_first_entry(&gl->gl_holders, struct gfs2_holder, gh_list);
1527 if (!(gh->gh_flags & LM_FLAG_PRIORITY)) {
1528 spin_unlock(&gl->gl_lockref.lock);
1529 if (sdp->sd_lockstruct.ls_ops->lm_cancel)
1530 sdp->sd_lockstruct.ls_ops->lm_cancel(gl);
1531 spin_lock(&gl->gl_lockref.lock);
1532 }
1533 return;
1534
1535trap_recursive:
1536 fs_err(sdp, "original: %pSR\n", (void *)gh2->gh_ip);
1537 fs_err(sdp, "pid: %d\n", pid_nr(gh2->gh_owner_pid));
1538 fs_err(sdp, "lock type: %d req lock state : %d\n",
1539 gh2->gh_gl->gl_name.ln_type, gh2->gh_state);
1540 fs_err(sdp, "new: %pSR\n", (void *)gh->gh_ip);
1541 fs_err(sdp, "pid: %d\n", pid_nr(gh->gh_owner_pid));
1542 fs_err(sdp, "lock type: %d req lock state : %d\n",
1543 gh->gh_gl->gl_name.ln_type, gh->gh_state);
1544 gfs2_dump_glock(NULL, gl, true);
1545 BUG();
1546}
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557int gfs2_glock_nq(struct gfs2_holder *gh)
1558{
1559 struct gfs2_glock *gl = gh->gh_gl;
1560 int error = 0;
1561
1562 if (glock_blocked_by_withdraw(gl) && !(gh->gh_flags & LM_FLAG_NOEXP))
1563 return -EIO;
1564
1565 if (test_bit(GLF_LRU, &gl->gl_flags))
1566 gfs2_glock_remove_from_lru(gl);
1567
1568 spin_lock(&gl->gl_lockref.lock);
1569 add_to_queue(gh);
1570 if (unlikely((LM_FLAG_NOEXP & gh->gh_flags) &&
1571 test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))) {
1572 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1573 gl->gl_lockref.count++;
1574 __gfs2_glock_queue_work(gl, 0);
1575 }
1576 run_queue(gl, 1);
1577 spin_unlock(&gl->gl_lockref.lock);
1578
1579 if (!(gh->gh_flags & GL_ASYNC))
1580 error = gfs2_glock_wait(gh);
1581
1582 return error;
1583}
1584
1585
1586
1587
1588
1589
1590
1591
1592int gfs2_glock_poll(struct gfs2_holder *gh)
1593{
1594 return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1;
1595}
1596
1597static inline bool needs_demote(struct gfs2_glock *gl)
1598{
1599 return (test_bit(GLF_DEMOTE, &gl->gl_flags) ||
1600 test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags));
1601}
1602
1603static void __gfs2_glock_dq(struct gfs2_holder *gh)
1604{
1605 struct gfs2_glock *gl = gh->gh_gl;
1606 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1607 unsigned delay = 0;
1608 int fast_path = 0;
1609
1610
1611
1612
1613
1614
1615
1616 while (gh) {
1617
1618
1619
1620
1621
1622
1623
1624 if (test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags) &&
1625 glock_blocked_by_withdraw(gl) &&
1626 gh->gh_gl != sdp->sd_jinode_gl) {
1627 sdp->sd_glock_dqs_held++;
1628 spin_unlock(&gl->gl_lockref.lock);
1629 might_sleep();
1630 wait_on_bit(&sdp->sd_flags, SDF_WITHDRAW_RECOVERY,
1631 TASK_UNINTERRUPTIBLE);
1632 spin_lock(&gl->gl_lockref.lock);
1633 }
1634
1635
1636
1637
1638
1639
1640 if (gh->gh_flags & GL_NOCACHE)
1641 handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1642
1643 list_del_init(&gh->gh_list);
1644 clear_bit(HIF_HOLDER, &gh->gh_iflags);
1645 trace_gfs2_glock_queue(gh, 0);
1646
1647
1648
1649
1650
1651 if (!needs_demote(gl)) {
1652 if (list_empty(&gl->gl_holders))
1653 fast_path = 1;
1654 break;
1655 }
1656
1657
1658
1659
1660 if (find_first_strong_holder(gl))
1661 break;
1662
1663
1664
1665
1666
1667
1668 gh = find_first_holder(gl);
1669 }
1670
1671 if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl))
1672 gfs2_glock_add_to_lru(gl);
1673
1674 if (unlikely(!fast_path)) {
1675 gl->gl_lockref.count++;
1676 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1677 !test_bit(GLF_DEMOTE, &gl->gl_flags) &&
1678 gl->gl_name.ln_type == LM_TYPE_INODE)
1679 delay = gl->gl_hold_time;
1680 __gfs2_glock_queue_work(gl, delay);
1681 }
1682}
1683
1684
1685
1686
1687
1688
1689void gfs2_glock_dq(struct gfs2_holder *gh)
1690{
1691 struct gfs2_glock *gl = gh->gh_gl;
1692
1693 spin_lock(&gl->gl_lockref.lock);
1694 __gfs2_glock_dq(gh);
1695 spin_unlock(&gl->gl_lockref.lock);
1696}
1697
1698void gfs2_glock_dq_wait(struct gfs2_holder *gh)
1699{
1700 struct gfs2_glock *gl = gh->gh_gl;
1701 gfs2_glock_dq(gh);
1702 might_sleep();
1703 wait_on_bit(&gl->gl_flags, GLF_DEMOTE, TASK_UNINTERRUPTIBLE);
1704}
1705
1706
1707
1708
1709
1710
1711
1712void gfs2_glock_dq_uninit(struct gfs2_holder *gh)
1713{
1714 gfs2_glock_dq(gh);
1715 gfs2_holder_uninit(gh);
1716}
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730int gfs2_glock_nq_num(struct gfs2_sbd *sdp, u64 number,
1731 const struct gfs2_glock_operations *glops,
1732 unsigned int state, u16 flags, struct gfs2_holder *gh)
1733{
1734 struct gfs2_glock *gl;
1735 int error;
1736
1737 error = gfs2_glock_get(sdp, number, glops, CREATE, &gl);
1738 if (!error) {
1739 error = gfs2_glock_nq_init(gl, state, flags, gh);
1740 gfs2_glock_put(gl);
1741 }
1742
1743 return error;
1744}
1745
1746
1747
1748
1749
1750
1751
1752
1753static int glock_compare(const void *arg_a, const void *arg_b)
1754{
1755 const struct gfs2_holder *gh_a = *(const struct gfs2_holder **)arg_a;
1756 const struct gfs2_holder *gh_b = *(const struct gfs2_holder **)arg_b;
1757 const struct lm_lockname *a = &gh_a->gh_gl->gl_name;
1758 const struct lm_lockname *b = &gh_b->gh_gl->gl_name;
1759
1760 if (a->ln_number > b->ln_number)
1761 return 1;
1762 if (a->ln_number < b->ln_number)
1763 return -1;
1764 BUG_ON(gh_a->gh_gl->gl_ops->go_type == gh_b->gh_gl->gl_ops->go_type);
1765 return 0;
1766}
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778static int nq_m_sync(unsigned int num_gh, struct gfs2_holder *ghs,
1779 struct gfs2_holder **p)
1780{
1781 unsigned int x;
1782 int error = 0;
1783
1784 for (x = 0; x < num_gh; x++)
1785 p[x] = &ghs[x];
1786
1787 sort(p, num_gh, sizeof(struct gfs2_holder *), glock_compare, NULL);
1788
1789 for (x = 0; x < num_gh; x++) {
1790 p[x]->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC);
1791
1792 error = gfs2_glock_nq(p[x]);
1793 if (error) {
1794 while (x--)
1795 gfs2_glock_dq(p[x]);
1796 break;
1797 }
1798 }
1799
1800 return error;
1801}
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813int gfs2_glock_nq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1814{
1815 struct gfs2_holder *tmp[4];
1816 struct gfs2_holder **pph = tmp;
1817 int error = 0;
1818
1819 switch(num_gh) {
1820 case 0:
1821 return 0;
1822 case 1:
1823 ghs->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC);
1824 return gfs2_glock_nq(ghs);
1825 default:
1826 if (num_gh <= 4)
1827 break;
1828 pph = kmalloc_array(num_gh, sizeof(struct gfs2_holder *),
1829 GFP_NOFS);
1830 if (!pph)
1831 return -ENOMEM;
1832 }
1833
1834 error = nq_m_sync(num_gh, ghs, pph);
1835
1836 if (pph != tmp)
1837 kfree(pph);
1838
1839 return error;
1840}
1841
1842
1843
1844
1845
1846
1847
1848
1849void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1850{
1851 while (num_gh--)
1852 gfs2_glock_dq(&ghs[num_gh]);
1853}
1854
1855void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state)
1856{
1857 unsigned long delay = 0;
1858 unsigned long holdtime;
1859 unsigned long now = jiffies;
1860
1861 gfs2_glock_hold(gl);
1862 spin_lock(&gl->gl_lockref.lock);
1863 holdtime = gl->gl_tchange + gl->gl_hold_time;
1864 if (!list_empty(&gl->gl_holders) &&
1865 gl->gl_name.ln_type == LM_TYPE_INODE) {
1866 if (time_before(now, holdtime))
1867 delay = holdtime - now;
1868 if (test_bit(GLF_REPLY_PENDING, &gl->gl_flags))
1869 delay = gl->gl_hold_time;
1870 }
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888 if (!find_first_strong_holder(gl)) {
1889 struct gfs2_holder mock_gh = {
1890 .gh_gl = gl,
1891 .gh_state = (state == LM_ST_UNLOCKED) ?
1892 LM_ST_EXCLUSIVE : state,
1893 .gh_iflags = BIT(HIF_HOLDER)
1894 };
1895
1896 demote_incompat_holders(gl, &mock_gh);
1897 }
1898 handle_callback(gl, state, delay, true);
1899 __gfs2_glock_queue_work(gl, delay);
1900 spin_unlock(&gl->gl_lockref.lock);
1901}
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914static int gfs2_should_freeze(const struct gfs2_glock *gl)
1915{
1916 const struct gfs2_holder *gh;
1917
1918 if (gl->gl_reply & ~LM_OUT_ST_MASK)
1919 return 0;
1920 if (gl->gl_target == LM_ST_UNLOCKED)
1921 return 0;
1922
1923 list_for_each_entry(gh, &gl->gl_holders, gh_list) {
1924 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
1925 continue;
1926 if (LM_FLAG_NOEXP & gh->gh_flags)
1927 return 0;
1928 }
1929
1930 return 1;
1931}
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942void gfs2_glock_complete(struct gfs2_glock *gl, int ret)
1943{
1944 struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct;
1945
1946 spin_lock(&gl->gl_lockref.lock);
1947 gl->gl_reply = ret;
1948
1949 if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags))) {
1950 if (gfs2_should_freeze(gl)) {
1951 set_bit(GLF_FROZEN, &gl->gl_flags);
1952 spin_unlock(&gl->gl_lockref.lock);
1953 return;
1954 }
1955 }
1956
1957 gl->gl_lockref.count++;
1958 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1959 __gfs2_glock_queue_work(gl, 0);
1960 spin_unlock(&gl->gl_lockref.lock);
1961}
1962
1963static int glock_cmp(void *priv, const struct list_head *a,
1964 const struct list_head *b)
1965{
1966 struct gfs2_glock *gla, *glb;
1967
1968 gla = list_entry(a, struct gfs2_glock, gl_lru);
1969 glb = list_entry(b, struct gfs2_glock, gl_lru);
1970
1971 if (gla->gl_name.ln_number > glb->gl_name.ln_number)
1972 return 1;
1973 if (gla->gl_name.ln_number < glb->gl_name.ln_number)
1974 return -1;
1975
1976 return 0;
1977}
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993static void gfs2_dispose_glock_lru(struct list_head *list)
1994__releases(&lru_lock)
1995__acquires(&lru_lock)
1996{
1997 struct gfs2_glock *gl;
1998
1999 list_sort(NULL, list, glock_cmp);
2000
2001 while(!list_empty(list)) {
2002 gl = list_first_entry(list, struct gfs2_glock, gl_lru);
2003 list_del_init(&gl->gl_lru);
2004 clear_bit(GLF_LRU, &gl->gl_flags);
2005 if (!spin_trylock(&gl->gl_lockref.lock)) {
2006add_back_to_lru:
2007 list_add(&gl->gl_lru, &lru_list);
2008 set_bit(GLF_LRU, &gl->gl_flags);
2009 atomic_inc(&lru_count);
2010 continue;
2011 }
2012 if (test_and_set_bit(GLF_LOCK, &gl->gl_flags)) {
2013 spin_unlock(&gl->gl_lockref.lock);
2014 goto add_back_to_lru;
2015 }
2016 gl->gl_lockref.count++;
2017 if (demote_ok(gl))
2018 handle_callback(gl, LM_ST_UNLOCKED, 0, false);
2019 WARN_ON(!test_and_clear_bit(GLF_LOCK, &gl->gl_flags));
2020 __gfs2_glock_queue_work(gl, 0);
2021 spin_unlock(&gl->gl_lockref.lock);
2022 cond_resched_lock(&lru_lock);
2023 }
2024}
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035static long gfs2_scan_glock_lru(int nr)
2036{
2037 struct gfs2_glock *gl;
2038 LIST_HEAD(skipped);
2039 LIST_HEAD(dispose);
2040 long freed = 0;
2041
2042 spin_lock(&lru_lock);
2043 while ((nr-- >= 0) && !list_empty(&lru_list)) {
2044 gl = list_first_entry(&lru_list, struct gfs2_glock, gl_lru);
2045
2046
2047 if (!test_bit(GLF_LOCK, &gl->gl_flags)) {
2048 list_move(&gl->gl_lru, &dispose);
2049 atomic_dec(&lru_count);
2050 freed++;
2051 continue;
2052 }
2053
2054 list_move(&gl->gl_lru, &skipped);
2055 }
2056 list_splice(&skipped, &lru_list);
2057 if (!list_empty(&dispose))
2058 gfs2_dispose_glock_lru(&dispose);
2059 spin_unlock(&lru_lock);
2060
2061 return freed;
2062}
2063
2064static unsigned long gfs2_glock_shrink_scan(struct shrinker *shrink,
2065 struct shrink_control *sc)
2066{
2067 if (!(sc->gfp_mask & __GFP_FS))
2068 return SHRINK_STOP;
2069 return gfs2_scan_glock_lru(sc->nr_to_scan);
2070}
2071
2072static unsigned long gfs2_glock_shrink_count(struct shrinker *shrink,
2073 struct shrink_control *sc)
2074{
2075 return vfs_pressure_ratio(atomic_read(&lru_count));
2076}
2077
2078static struct shrinker glock_shrinker = {
2079 .seeks = DEFAULT_SEEKS,
2080 .count_objects = gfs2_glock_shrink_count,
2081 .scan_objects = gfs2_glock_shrink_scan,
2082};
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp)
2095{
2096 struct gfs2_glock *gl;
2097 struct rhashtable_iter iter;
2098
2099 rhashtable_walk_enter(&gl_hash_table, &iter);
2100
2101 do {
2102 rhashtable_walk_start(&iter);
2103
2104 while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl)) {
2105 if (gl->gl_name.ln_sbd == sdp)
2106 examiner(gl);
2107 }
2108
2109 rhashtable_walk_stop(&iter);
2110 } while (cond_resched(), gl == ERR_PTR(-EAGAIN));
2111
2112 rhashtable_walk_exit(&iter);
2113}
2114
2115bool gfs2_queue_delete_work(struct gfs2_glock *gl, unsigned long delay)
2116{
2117 bool queued;
2118
2119 spin_lock(&gl->gl_lockref.lock);
2120 queued = queue_delayed_work(gfs2_delete_workqueue,
2121 &gl->gl_delete, delay);
2122 if (queued)
2123 set_bit(GLF_PENDING_DELETE, &gl->gl_flags);
2124 spin_unlock(&gl->gl_lockref.lock);
2125 return queued;
2126}
2127
2128void gfs2_cancel_delete_work(struct gfs2_glock *gl)
2129{
2130 if (cancel_delayed_work(&gl->gl_delete)) {
2131 clear_bit(GLF_PENDING_DELETE, &gl->gl_flags);
2132 gfs2_glock_put(gl);
2133 }
2134}
2135
2136bool gfs2_delete_work_queued(const struct gfs2_glock *gl)
2137{
2138 return test_bit(GLF_PENDING_DELETE, &gl->gl_flags);
2139}
2140
2141static void flush_delete_work(struct gfs2_glock *gl)
2142{
2143 if (gl->gl_name.ln_type == LM_TYPE_IOPEN) {
2144 if (cancel_delayed_work(&gl->gl_delete)) {
2145 queue_delayed_work(gfs2_delete_workqueue,
2146 &gl->gl_delete, 0);
2147 }
2148 }
2149}
2150
2151void gfs2_flush_delete_work(struct gfs2_sbd *sdp)
2152{
2153 glock_hash_walk(flush_delete_work, sdp);
2154 flush_workqueue(gfs2_delete_workqueue);
2155}
2156
2157
2158
2159
2160
2161
2162
2163static void thaw_glock(struct gfs2_glock *gl)
2164{
2165 if (!test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))
2166 return;
2167 if (!lockref_get_not_dead(&gl->gl_lockref))
2168 return;
2169 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
2170 gfs2_glock_queue_work(gl, 0);
2171}
2172
2173
2174
2175
2176
2177
2178
2179static void clear_glock(struct gfs2_glock *gl)
2180{
2181 gfs2_glock_remove_from_lru(gl);
2182
2183 spin_lock(&gl->gl_lockref.lock);
2184 if (!__lockref_is_dead(&gl->gl_lockref)) {
2185 gl->gl_lockref.count++;
2186 if (gl->gl_state != LM_ST_UNLOCKED)
2187 handle_callback(gl, LM_ST_UNLOCKED, 0, false);
2188 __gfs2_glock_queue_work(gl, 0);
2189 }
2190 spin_unlock(&gl->gl_lockref.lock);
2191}
2192
2193
2194
2195
2196
2197
2198
2199void gfs2_glock_thaw(struct gfs2_sbd *sdp)
2200{
2201 glock_hash_walk(thaw_glock, sdp);
2202}
2203
2204static void dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
2205{
2206 spin_lock(&gl->gl_lockref.lock);
2207 gfs2_dump_glock(seq, gl, fsid);
2208 spin_unlock(&gl->gl_lockref.lock);
2209}
2210
2211static void dump_glock_func(struct gfs2_glock *gl)
2212{
2213 dump_glock(NULL, gl, true);
2214}
2215
2216
2217
2218
2219
2220
2221
2222
2223void gfs2_gl_hash_clear(struct gfs2_sbd *sdp)
2224{
2225 set_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags);
2226 flush_workqueue(glock_workqueue);
2227 glock_hash_walk(clear_glock, sdp);
2228 flush_workqueue(glock_workqueue);
2229 wait_event_timeout(sdp->sd_glock_wait,
2230 atomic_read(&sdp->sd_glock_disposal) == 0,
2231 HZ * 600);
2232 glock_hash_walk(dump_glock_func, sdp);
2233}
2234
2235void gfs2_glock_finish_truncate(struct gfs2_inode *ip)
2236{
2237 struct gfs2_glock *gl = ip->i_gl;
2238 int ret;
2239
2240 ret = gfs2_truncatei_resume(ip);
2241 gfs2_glock_assert_withdraw(gl, ret == 0);
2242
2243 spin_lock(&gl->gl_lockref.lock);
2244 clear_bit(GLF_LOCK, &gl->gl_flags);
2245 run_queue(gl, 1);
2246 spin_unlock(&gl->gl_lockref.lock);
2247}
2248
2249static const char *state2str(unsigned state)
2250{
2251 switch(state) {
2252 case LM_ST_UNLOCKED:
2253 return "UN";
2254 case LM_ST_SHARED:
2255 return "SH";
2256 case LM_ST_DEFERRED:
2257 return "DF";
2258 case LM_ST_EXCLUSIVE:
2259 return "EX";
2260 }
2261 return "??";
2262}
2263
2264static const char *hflags2str(char *buf, u16 flags, unsigned long iflags)
2265{
2266 char *p = buf;
2267 if (flags & LM_FLAG_TRY)
2268 *p++ = 't';
2269 if (flags & LM_FLAG_TRY_1CB)
2270 *p++ = 'T';
2271 if (flags & LM_FLAG_NOEXP)
2272 *p++ = 'e';
2273 if (flags & LM_FLAG_ANY)
2274 *p++ = 'A';
2275 if (flags & LM_FLAG_PRIORITY)
2276 *p++ = 'p';
2277 if (flags & LM_FLAG_NODE_SCOPE)
2278 *p++ = 'n';
2279 if (flags & GL_ASYNC)
2280 *p++ = 'a';
2281 if (flags & GL_EXACT)
2282 *p++ = 'E';
2283 if (flags & GL_NOCACHE)
2284 *p++ = 'c';
2285 if (test_bit(HIF_HOLDER, &iflags))
2286 *p++ = 'H';
2287 if (test_bit(HIF_WAIT, &iflags))
2288 *p++ = 'W';
2289 if (test_bit(HIF_MAY_DEMOTE, &iflags))
2290 *p++ = 'D';
2291 if (flags & GL_SKIP)
2292 *p++ = 's';
2293 *p = 0;
2294 return buf;
2295}
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh,
2306 const char *fs_id_buf)
2307{
2308 struct task_struct *gh_owner = NULL;
2309 char flags_buf[32];
2310
2311 rcu_read_lock();
2312 if (gh->gh_owner_pid)
2313 gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID);
2314 gfs2_print_dbg(seq, "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n",
2315 fs_id_buf, state2str(gh->gh_state),
2316 hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags),
2317 gh->gh_error,
2318 gh->gh_owner_pid ? (long)pid_nr(gh->gh_owner_pid) : -1,
2319 gh_owner ? gh_owner->comm : "(ended)",
2320 (void *)gh->gh_ip);
2321 rcu_read_unlock();
2322}
2323
2324static const char *gflags2str(char *buf, const struct gfs2_glock *gl)
2325{
2326 const unsigned long *gflags = &gl->gl_flags;
2327 char *p = buf;
2328
2329 if (test_bit(GLF_LOCK, gflags))
2330 *p++ = 'l';
2331 if (test_bit(GLF_DEMOTE, gflags))
2332 *p++ = 'D';
2333 if (test_bit(GLF_PENDING_DEMOTE, gflags))
2334 *p++ = 'd';
2335 if (test_bit(GLF_DEMOTE_IN_PROGRESS, gflags))
2336 *p++ = 'p';
2337 if (test_bit(GLF_DIRTY, gflags))
2338 *p++ = 'y';
2339 if (test_bit(GLF_LFLUSH, gflags))
2340 *p++ = 'f';
2341 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, gflags))
2342 *p++ = 'i';
2343 if (test_bit(GLF_REPLY_PENDING, gflags))
2344 *p++ = 'r';
2345 if (test_bit(GLF_INITIAL, gflags))
2346 *p++ = 'I';
2347 if (test_bit(GLF_FROZEN, gflags))
2348 *p++ = 'F';
2349 if (!list_empty(&gl->gl_holders))
2350 *p++ = 'q';
2351 if (test_bit(GLF_LRU, gflags))
2352 *p++ = 'L';
2353 if (gl->gl_object)
2354 *p++ = 'o';
2355 if (test_bit(GLF_BLOCKING, gflags))
2356 *p++ = 'b';
2357 if (test_bit(GLF_PENDING_DELETE, gflags))
2358 *p++ = 'P';
2359 if (test_bit(GLF_FREEING, gflags))
2360 *p++ = 'x';
2361 if (test_bit(GLF_INSTANTIATE_NEEDED, gflags))
2362 *p++ = 'n';
2363 if (test_bit(GLF_INSTANTIATE_IN_PROG, gflags))
2364 *p++ = 'N';
2365 *p = 0;
2366 return buf;
2367}
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
2388{
2389 const struct gfs2_glock_operations *glops = gl->gl_ops;
2390 unsigned long long dtime;
2391 const struct gfs2_holder *gh;
2392 char gflags_buf[32];
2393 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
2394 char fs_id_buf[sizeof(sdp->sd_fsname) + 7];
2395 unsigned long nrpages = 0;
2396
2397 if (gl->gl_ops->go_flags & GLOF_ASPACE) {
2398 struct address_space *mapping = gfs2_glock2aspace(gl);
2399
2400 nrpages = mapping->nrpages;
2401 }
2402 memset(fs_id_buf, 0, sizeof(fs_id_buf));
2403 if (fsid && sdp)
2404 sprintf(fs_id_buf, "fsid=%s: ", sdp->sd_fsname);
2405 dtime = jiffies - gl->gl_demote_time;
2406 dtime *= 1000000/HZ;
2407 if (!test_bit(GLF_DEMOTE, &gl->gl_flags))
2408 dtime = 0;
2409 gfs2_print_dbg(seq, "%sG: s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d "
2410 "v:%d r:%d m:%ld p:%lu\n",
2411 fs_id_buf, state2str(gl->gl_state),
2412 gl->gl_name.ln_type,
2413 (unsigned long long)gl->gl_name.ln_number,
2414 gflags2str(gflags_buf, gl),
2415 state2str(gl->gl_target),
2416 state2str(gl->gl_demote_state), dtime,
2417 atomic_read(&gl->gl_ail_count),
2418 atomic_read(&gl->gl_revokes),
2419 (int)gl->gl_lockref.count, gl->gl_hold_time, nrpages);
2420
2421 list_for_each_entry(gh, &gl->gl_holders, gh_list)
2422 dump_holder(seq, gh, fs_id_buf);
2423
2424 if (gl->gl_state != LM_ST_UNLOCKED && glops->go_dump)
2425 glops->go_dump(seq, gl, fs_id_buf);
2426}
2427
2428static int gfs2_glstats_seq_show(struct seq_file *seq, void *iter_ptr)
2429{
2430 struct gfs2_glock *gl = iter_ptr;
2431
2432 seq_printf(seq, "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n",
2433 gl->gl_name.ln_type,
2434 (unsigned long long)gl->gl_name.ln_number,
2435 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT],
2436 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR],
2437 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB],
2438 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVARB],
2439 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRT],
2440 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRTVAR],
2441 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_DCOUNT],
2442 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_QCOUNT]);
2443 return 0;
2444}
2445
2446static const char *gfs2_gltype[] = {
2447 "type",
2448 "reserved",
2449 "nondisk",
2450 "inode",
2451 "rgrp",
2452 "meta",
2453 "iopen",
2454 "flock",
2455 "plock",
2456 "quota",
2457 "journal",
2458};
2459
2460static const char *gfs2_stype[] = {
2461 [GFS2_LKS_SRTT] = "srtt",
2462 [GFS2_LKS_SRTTVAR] = "srttvar",
2463 [GFS2_LKS_SRTTB] = "srttb",
2464 [GFS2_LKS_SRTTVARB] = "srttvarb",
2465 [GFS2_LKS_SIRT] = "sirt",
2466 [GFS2_LKS_SIRTVAR] = "sirtvar",
2467 [GFS2_LKS_DCOUNT] = "dlm",
2468 [GFS2_LKS_QCOUNT] = "queue",
2469};
2470
2471#define GFS2_NR_SBSTATS (ARRAY_SIZE(gfs2_gltype) * ARRAY_SIZE(gfs2_stype))
2472
2473static int gfs2_sbstats_seq_show(struct seq_file *seq, void *iter_ptr)
2474{
2475 struct gfs2_sbd *sdp = seq->private;
2476 loff_t pos = *(loff_t *)iter_ptr;
2477 unsigned index = pos >> 3;
2478 unsigned subindex = pos & 0x07;
2479 int i;
2480
2481 if (index == 0 && subindex != 0)
2482 return 0;
2483
2484 seq_printf(seq, "%-10s %8s:", gfs2_gltype[index],
2485 (index == 0) ? "cpu": gfs2_stype[subindex]);
2486
2487 for_each_possible_cpu(i) {
2488 const struct gfs2_pcpu_lkstats *lkstats = per_cpu_ptr(sdp->sd_lkstats, i);
2489
2490 if (index == 0)
2491 seq_printf(seq, " %15u", i);
2492 else
2493 seq_printf(seq, " %15llu", (unsigned long long)lkstats->
2494 lkstats[index - 1].stats[subindex]);
2495 }
2496 seq_putc(seq, '\n');
2497 return 0;
2498}
2499
2500int __init gfs2_glock_init(void)
2501{
2502 int i, ret;
2503
2504 ret = rhashtable_init(&gl_hash_table, &ht_parms);
2505 if (ret < 0)
2506 return ret;
2507
2508 glock_workqueue = alloc_workqueue("glock_workqueue", WQ_MEM_RECLAIM |
2509 WQ_HIGHPRI | WQ_FREEZABLE, 0);
2510 if (!glock_workqueue) {
2511 rhashtable_destroy(&gl_hash_table);
2512 return -ENOMEM;
2513 }
2514 gfs2_delete_workqueue = alloc_workqueue("delete_workqueue",
2515 WQ_MEM_RECLAIM | WQ_FREEZABLE,
2516 0);
2517 if (!gfs2_delete_workqueue) {
2518 destroy_workqueue(glock_workqueue);
2519 rhashtable_destroy(&gl_hash_table);
2520 return -ENOMEM;
2521 }
2522
2523 ret = register_shrinker(&glock_shrinker);
2524 if (ret) {
2525 destroy_workqueue(gfs2_delete_workqueue);
2526 destroy_workqueue(glock_workqueue);
2527 rhashtable_destroy(&gl_hash_table);
2528 return ret;
2529 }
2530
2531 for (i = 0; i < GLOCK_WAIT_TABLE_SIZE; i++)
2532 init_waitqueue_head(glock_wait_table + i);
2533
2534 return 0;
2535}
2536
2537void gfs2_glock_exit(void)
2538{
2539 unregister_shrinker(&glock_shrinker);
2540 rhashtable_destroy(&gl_hash_table);
2541 destroy_workqueue(glock_workqueue);
2542 destroy_workqueue(gfs2_delete_workqueue);
2543}
2544
2545static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n)
2546{
2547 struct gfs2_glock *gl = gi->gl;
2548
2549 if (gl) {
2550 if (n == 0)
2551 return;
2552 if (!lockref_put_not_zero(&gl->gl_lockref))
2553 gfs2_glock_queue_put(gl);
2554 }
2555 for (;;) {
2556 gl = rhashtable_walk_next(&gi->hti);
2557 if (IS_ERR_OR_NULL(gl)) {
2558 if (gl == ERR_PTR(-EAGAIN)) {
2559 n = 1;
2560 continue;
2561 }
2562 gl = NULL;
2563 break;
2564 }
2565 if (gl->gl_name.ln_sbd != gi->sdp)
2566 continue;
2567 if (n <= 1) {
2568 if (!lockref_get_not_dead(&gl->gl_lockref))
2569 continue;
2570 break;
2571 } else {
2572 if (__lockref_is_dead(&gl->gl_lockref))
2573 continue;
2574 n--;
2575 }
2576 }
2577 gi->gl = gl;
2578}
2579
2580static void *gfs2_glock_seq_start(struct seq_file *seq, loff_t *pos)
2581 __acquires(RCU)
2582{
2583 struct gfs2_glock_iter *gi = seq->private;
2584 loff_t n;
2585
2586
2587
2588
2589
2590 if (*pos < gi->last_pos) {
2591 rhashtable_walk_exit(&gi->hti);
2592 rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2593 n = *pos + 1;
2594 } else {
2595 n = *pos - gi->last_pos;
2596 }
2597
2598 rhashtable_walk_start(&gi->hti);
2599
2600 gfs2_glock_iter_next(gi, n);
2601 gi->last_pos = *pos;
2602 return gi->gl;
2603}
2604
2605static void *gfs2_glock_seq_next(struct seq_file *seq, void *iter_ptr,
2606 loff_t *pos)
2607{
2608 struct gfs2_glock_iter *gi = seq->private;
2609
2610 (*pos)++;
2611 gi->last_pos = *pos;
2612 gfs2_glock_iter_next(gi, 1);
2613 return gi->gl;
2614}
2615
2616static void gfs2_glock_seq_stop(struct seq_file *seq, void *iter_ptr)
2617 __releases(RCU)
2618{
2619 struct gfs2_glock_iter *gi = seq->private;
2620
2621 rhashtable_walk_stop(&gi->hti);
2622}
2623
2624static int gfs2_glock_seq_show(struct seq_file *seq, void *iter_ptr)
2625{
2626 dump_glock(seq, iter_ptr, false);
2627 return 0;
2628}
2629
2630static void *gfs2_sbstats_seq_start(struct seq_file *seq, loff_t *pos)
2631{
2632 preempt_disable();
2633 if (*pos >= GFS2_NR_SBSTATS)
2634 return NULL;
2635 return pos;
2636}
2637
2638static void *gfs2_sbstats_seq_next(struct seq_file *seq, void *iter_ptr,
2639 loff_t *pos)
2640{
2641 (*pos)++;
2642 if (*pos >= GFS2_NR_SBSTATS)
2643 return NULL;
2644 return pos;
2645}
2646
2647static void gfs2_sbstats_seq_stop(struct seq_file *seq, void *iter_ptr)
2648{
2649 preempt_enable();
2650}
2651
2652static const struct seq_operations gfs2_glock_seq_ops = {
2653 .start = gfs2_glock_seq_start,
2654 .next = gfs2_glock_seq_next,
2655 .stop = gfs2_glock_seq_stop,
2656 .show = gfs2_glock_seq_show,
2657};
2658
2659static const struct seq_operations gfs2_glstats_seq_ops = {
2660 .start = gfs2_glock_seq_start,
2661 .next = gfs2_glock_seq_next,
2662 .stop = gfs2_glock_seq_stop,
2663 .show = gfs2_glstats_seq_show,
2664};
2665
2666static const struct seq_operations gfs2_sbstats_sops = {
2667 .start = gfs2_sbstats_seq_start,
2668 .next = gfs2_sbstats_seq_next,
2669 .stop = gfs2_sbstats_seq_stop,
2670 .show = gfs2_sbstats_seq_show,
2671};
2672
2673#define GFS2_SEQ_GOODSIZE min(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER, 65536UL)
2674
2675static int __gfs2_glocks_open(struct inode *inode, struct file *file,
2676 const struct seq_operations *ops)
2677{
2678 int ret = seq_open_private(file, ops, sizeof(struct gfs2_glock_iter));
2679 if (ret == 0) {
2680 struct seq_file *seq = file->private_data;
2681 struct gfs2_glock_iter *gi = seq->private;
2682
2683 gi->sdp = inode->i_private;
2684 seq->buf = kmalloc(GFS2_SEQ_GOODSIZE, GFP_KERNEL | __GFP_NOWARN);
2685 if (seq->buf)
2686 seq->size = GFS2_SEQ_GOODSIZE;
2687
2688
2689
2690
2691 gi->last_pos = -1;
2692 gi->gl = NULL;
2693 rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2694 }
2695 return ret;
2696}
2697
2698static int gfs2_glocks_open(struct inode *inode, struct file *file)
2699{
2700 return __gfs2_glocks_open(inode, file, &gfs2_glock_seq_ops);
2701}
2702
2703static int gfs2_glocks_release(struct inode *inode, struct file *file)
2704{
2705 struct seq_file *seq = file->private_data;
2706 struct gfs2_glock_iter *gi = seq->private;
2707
2708 if (gi->gl)
2709 gfs2_glock_put(gi->gl);
2710 rhashtable_walk_exit(&gi->hti);
2711 return seq_release_private(inode, file);
2712}
2713
2714static int gfs2_glstats_open(struct inode *inode, struct file *file)
2715{
2716 return __gfs2_glocks_open(inode, file, &gfs2_glstats_seq_ops);
2717}
2718
2719static const struct file_operations gfs2_glocks_fops = {
2720 .owner = THIS_MODULE,
2721 .open = gfs2_glocks_open,
2722 .read = seq_read,
2723 .llseek = seq_lseek,
2724 .release = gfs2_glocks_release,
2725};
2726
2727static const struct file_operations gfs2_glstats_fops = {
2728 .owner = THIS_MODULE,
2729 .open = gfs2_glstats_open,
2730 .read = seq_read,
2731 .llseek = seq_lseek,
2732 .release = gfs2_glocks_release,
2733};
2734
2735DEFINE_SEQ_ATTRIBUTE(gfs2_sbstats);
2736
2737void gfs2_create_debugfs_file(struct gfs2_sbd *sdp)
2738{
2739 sdp->debugfs_dir = debugfs_create_dir(sdp->sd_table_name, gfs2_root);
2740
2741 debugfs_create_file("glocks", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2742 &gfs2_glocks_fops);
2743
2744 debugfs_create_file("glstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2745 &gfs2_glstats_fops);
2746
2747 debugfs_create_file("sbstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2748 &gfs2_sbstats_fops);
2749}
2750
2751void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp)
2752{
2753 debugfs_remove_recursive(sdp->debugfs_dir);
2754 sdp->debugfs_dir = NULL;
2755}
2756
2757void gfs2_register_debugfs(void)
2758{
2759 gfs2_root = debugfs_create_dir("gfs2", NULL);
2760}
2761
2762void gfs2_unregister_debugfs(void)
2763{
2764 debugfs_remove(gfs2_root);
2765 gfs2_root = NULL;
2766}
2767