1
2
3
4
5
6
7#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9#include <linux/sched.h>
10#include <linux/slab.h>
11#include <linux/spinlock.h>
12#include <linux/buffer_head.h>
13#include <linux/delay.h>
14#include <linux/sort.h>
15#include <linux/hash.h>
16#include <linux/jhash.h>
17#include <linux/kallsyms.h>
18#include <linux/gfs2_ondisk.h>
19#include <linux/list.h>
20#include <linux/wait.h>
21#include <linux/module.h>
22#include <linux/uaccess.h>
23#include <linux/seq_file.h>
24#include <linux/debugfs.h>
25#include <linux/kthread.h>
26#include <linux/freezer.h>
27#include <linux/workqueue.h>
28#include <linux/jiffies.h>
29#include <linux/rcupdate.h>
30#include <linux/rculist_bl.h>
31#include <linux/bit_spinlock.h>
32#include <linux/percpu.h>
33#include <linux/list_sort.h>
34#include <linux/lockref.h>
35#include <linux/rhashtable.h>
36
37#include "gfs2.h"
38#include "incore.h"
39#include "glock.h"
40#include "glops.h"
41#include "inode.h"
42#include "lops.h"
43#include "meta_io.h"
44#include "quota.h"
45#include "super.h"
46#include "util.h"
47#include "bmap.h"
48#define CREATE_TRACE_POINTS
49#include "trace_gfs2.h"
50
51struct gfs2_glock_iter {
52 struct gfs2_sbd *sdp;
53 struct rhashtable_iter hti;
54 struct gfs2_glock *gl;
55 loff_t last_pos;
56};
57
58typedef void (*glock_examiner) (struct gfs2_glock * gl);
59
60static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target);
61
62static struct dentry *gfs2_root;
63static struct workqueue_struct *glock_workqueue;
64struct workqueue_struct *gfs2_delete_workqueue;
65static LIST_HEAD(lru_list);
66static atomic_t lru_count = ATOMIC_INIT(0);
67static DEFINE_SPINLOCK(lru_lock);
68
69#define GFS2_GL_HASH_SHIFT 15
70#define GFS2_GL_HASH_SIZE BIT(GFS2_GL_HASH_SHIFT)
71
72static const struct rhashtable_params ht_parms = {
73 .nelem_hint = GFS2_GL_HASH_SIZE * 3 / 4,
74 .key_len = offsetofend(struct lm_lockname, ln_type),
75 .key_offset = offsetof(struct gfs2_glock, gl_name),
76 .head_offset = offsetof(struct gfs2_glock, gl_node),
77};
78
79static struct rhashtable gl_hash_table;
80
81#define GLOCK_WAIT_TABLE_BITS 12
82#define GLOCK_WAIT_TABLE_SIZE (1 << GLOCK_WAIT_TABLE_BITS)
83static wait_queue_head_t glock_wait_table[GLOCK_WAIT_TABLE_SIZE] __cacheline_aligned;
84
85struct wait_glock_queue {
86 struct lm_lockname *name;
87 wait_queue_entry_t wait;
88};
89
90static int glock_wake_function(wait_queue_entry_t *wait, unsigned int mode,
91 int sync, void *key)
92{
93 struct wait_glock_queue *wait_glock =
94 container_of(wait, struct wait_glock_queue, wait);
95 struct lm_lockname *wait_name = wait_glock->name;
96 struct lm_lockname *wake_name = key;
97
98 if (wake_name->ln_sbd != wait_name->ln_sbd ||
99 wake_name->ln_number != wait_name->ln_number ||
100 wake_name->ln_type != wait_name->ln_type)
101 return 0;
102 return autoremove_wake_function(wait, mode, sync, key);
103}
104
105static wait_queue_head_t *glock_waitqueue(struct lm_lockname *name)
106{
107 u32 hash = jhash2((u32 *)name, ht_parms.key_len / 4, 0);
108
109 return glock_wait_table + hash_32(hash, GLOCK_WAIT_TABLE_BITS);
110}
111
112
113
114
115
116static void wake_up_glock(struct gfs2_glock *gl)
117{
118 wait_queue_head_t *wq = glock_waitqueue(&gl->gl_name);
119
120 if (waitqueue_active(wq))
121 __wake_up(wq, TASK_NORMAL, 1, &gl->gl_name);
122}
123
124static void gfs2_glock_dealloc(struct rcu_head *rcu)
125{
126 struct gfs2_glock *gl = container_of(rcu, struct gfs2_glock, gl_rcu);
127
128 kfree(gl->gl_lksb.sb_lvbptr);
129 if (gl->gl_ops->go_flags & GLOF_ASPACE)
130 kmem_cache_free(gfs2_glock_aspace_cachep, gl);
131 else
132 kmem_cache_free(gfs2_glock_cachep, gl);
133}
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148static bool glock_blocked_by_withdraw(struct gfs2_glock *gl)
149{
150 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
151
152 if (likely(!gfs2_withdrawn(sdp)))
153 return false;
154 if (gl->gl_ops->go_flags & GLOF_NONDISK)
155 return false;
156 if (!sdp->sd_jdesc ||
157 gl->gl_name.ln_number == sdp->sd_jdesc->jd_no_addr)
158 return false;
159 return true;
160}
161
162void gfs2_glock_free(struct gfs2_glock *gl)
163{
164 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
165
166 gfs2_glock_assert_withdraw(gl, atomic_read(&gl->gl_revokes) == 0);
167 rhashtable_remove_fast(&gl_hash_table, &gl->gl_node, ht_parms);
168 smp_mb();
169 wake_up_glock(gl);
170 call_rcu(&gl->gl_rcu, gfs2_glock_dealloc);
171 if (atomic_dec_and_test(&sdp->sd_glock_disposal))
172 wake_up(&sdp->sd_glock_wait);
173}
174
175
176
177
178
179
180
181void gfs2_glock_hold(struct gfs2_glock *gl)
182{
183 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
184 lockref_get(&gl->gl_lockref);
185}
186
187
188
189
190
191
192
193
194static int demote_ok(const struct gfs2_glock *gl)
195{
196 const struct gfs2_glock_operations *glops = gl->gl_ops;
197
198 if (gl->gl_state == LM_ST_UNLOCKED)
199 return 0;
200 if (!list_empty(&gl->gl_holders))
201 return 0;
202 if (glops->go_demote_ok)
203 return glops->go_demote_ok(gl);
204 return 1;
205}
206
207
208void gfs2_glock_add_to_lru(struct gfs2_glock *gl)
209{
210 if (!(gl->gl_ops->go_flags & GLOF_LRU))
211 return;
212
213 spin_lock(&lru_lock);
214
215 list_del(&gl->gl_lru);
216 list_add_tail(&gl->gl_lru, &lru_list);
217
218 if (!test_bit(GLF_LRU, &gl->gl_flags)) {
219 set_bit(GLF_LRU, &gl->gl_flags);
220 atomic_inc(&lru_count);
221 }
222
223 spin_unlock(&lru_lock);
224}
225
226static void gfs2_glock_remove_from_lru(struct gfs2_glock *gl)
227{
228 if (!(gl->gl_ops->go_flags & GLOF_LRU))
229 return;
230
231 spin_lock(&lru_lock);
232 if (test_bit(GLF_LRU, &gl->gl_flags)) {
233 list_del_init(&gl->gl_lru);
234 atomic_dec(&lru_count);
235 clear_bit(GLF_LRU, &gl->gl_flags);
236 }
237 spin_unlock(&lru_lock);
238}
239
240
241
242
243
244static void __gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
245 if (!queue_delayed_work(glock_workqueue, &gl->gl_work, delay)) {
246
247
248
249
250
251
252 GLOCK_BUG_ON(gl, gl->gl_lockref.count < 2);
253 gl->gl_lockref.count--;
254 }
255}
256
257static void gfs2_glock_queue_work(struct gfs2_glock *gl, unsigned long delay) {
258 spin_lock(&gl->gl_lockref.lock);
259 __gfs2_glock_queue_work(gl, delay);
260 spin_unlock(&gl->gl_lockref.lock);
261}
262
263static void __gfs2_glock_put(struct gfs2_glock *gl)
264{
265 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
266 struct address_space *mapping = gfs2_glock2aspace(gl);
267
268 lockref_mark_dead(&gl->gl_lockref);
269
270 gfs2_glock_remove_from_lru(gl);
271 spin_unlock(&gl->gl_lockref.lock);
272 GLOCK_BUG_ON(gl, !list_empty(&gl->gl_holders));
273 GLOCK_BUG_ON(gl, mapping && mapping->nrpages && !gfs2_withdrawn(sdp));
274 trace_gfs2_glock_put(gl);
275 sdp->sd_lockstruct.ls_ops->lm_put_lock(gl);
276}
277
278
279
280
281void gfs2_glock_queue_put(struct gfs2_glock *gl)
282{
283 gfs2_glock_queue_work(gl, 0);
284}
285
286
287
288
289
290
291
292void gfs2_glock_put(struct gfs2_glock *gl)
293{
294 if (lockref_put_or_lock(&gl->gl_lockref))
295 return;
296
297 __gfs2_glock_put(gl);
298}
299
300
301
302
303
304
305
306
307
308static inline int may_grant(const struct gfs2_glock *gl, const struct gfs2_holder *gh)
309{
310 const struct gfs2_holder *gh_head = list_first_entry(&gl->gl_holders, const struct gfs2_holder, gh_list);
311 if ((gh->gh_state == LM_ST_EXCLUSIVE ||
312 gh_head->gh_state == LM_ST_EXCLUSIVE) && gh != gh_head)
313 return 0;
314 if (gl->gl_state == gh->gh_state)
315 return 1;
316 if (gh->gh_flags & GL_EXACT)
317 return 0;
318 if (gl->gl_state == LM_ST_EXCLUSIVE) {
319 if (gh->gh_state == LM_ST_SHARED && gh_head->gh_state == LM_ST_SHARED)
320 return 1;
321 if (gh->gh_state == LM_ST_DEFERRED && gh_head->gh_state == LM_ST_DEFERRED)
322 return 1;
323 }
324 if (gl->gl_state != LM_ST_UNLOCKED && (gh->gh_flags & LM_FLAG_ANY))
325 return 1;
326 return 0;
327}
328
329static void gfs2_holder_wake(struct gfs2_holder *gh)
330{
331 clear_bit(HIF_WAIT, &gh->gh_iflags);
332 smp_mb__after_atomic();
333 wake_up_bit(&gh->gh_iflags, HIF_WAIT);
334 if (gh->gh_flags & GL_ASYNC) {
335 struct gfs2_sbd *sdp = gh->gh_gl->gl_name.ln_sbd;
336
337 wake_up(&sdp->sd_async_glock_wait);
338 }
339}
340
341
342
343
344
345
346static void do_error(struct gfs2_glock *gl, const int ret)
347{
348 struct gfs2_holder *gh, *tmp;
349
350 list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
351 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
352 continue;
353 if (ret & LM_OUT_ERROR)
354 gh->gh_error = -EIO;
355 else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))
356 gh->gh_error = GLR_TRYFAILED;
357 else
358 continue;
359 list_del_init(&gh->gh_list);
360 trace_gfs2_glock_queue(gh, 0);
361 gfs2_holder_wake(gh);
362 }
363}
364
365
366
367
368
369
370
371
372
373static int do_promote(struct gfs2_glock *gl)
374__releases(&gl->gl_lockref.lock)
375__acquires(&gl->gl_lockref.lock)
376{
377 const struct gfs2_glock_operations *glops = gl->gl_ops;
378 struct gfs2_holder *gh, *tmp;
379 int ret;
380
381restart:
382 list_for_each_entry_safe(gh, tmp, &gl->gl_holders, gh_list) {
383 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
384 continue;
385 if (may_grant(gl, gh)) {
386 if (gh->gh_list.prev == &gl->gl_holders &&
387 glops->go_lock) {
388 spin_unlock(&gl->gl_lockref.lock);
389
390 ret = glops->go_lock(gh);
391 spin_lock(&gl->gl_lockref.lock);
392 if (ret) {
393 if (ret == 1)
394 return 2;
395 gh->gh_error = ret;
396 list_del_init(&gh->gh_list);
397 trace_gfs2_glock_queue(gh, 0);
398 gfs2_holder_wake(gh);
399 goto restart;
400 }
401 set_bit(HIF_HOLDER, &gh->gh_iflags);
402 trace_gfs2_promote(gh, 1);
403 gfs2_holder_wake(gh);
404 goto restart;
405 }
406 set_bit(HIF_HOLDER, &gh->gh_iflags);
407 trace_gfs2_promote(gh, 0);
408 gfs2_holder_wake(gh);
409 continue;
410 }
411 if (gh->gh_list.prev == &gl->gl_holders)
412 return 1;
413 do_error(gl, 0);
414 break;
415 }
416 return 0;
417}
418
419
420
421
422
423
424static inline struct gfs2_holder *find_first_waiter(const struct gfs2_glock *gl)
425{
426 struct gfs2_holder *gh;
427
428 list_for_each_entry(gh, &gl->gl_holders, gh_list) {
429 if (!test_bit(HIF_HOLDER, &gh->gh_iflags))
430 return gh;
431 }
432 return NULL;
433}
434
435
436
437
438
439
440
441
442static void state_change(struct gfs2_glock *gl, unsigned int new_state)
443{
444 int held1, held2;
445
446 held1 = (gl->gl_state != LM_ST_UNLOCKED);
447 held2 = (new_state != LM_ST_UNLOCKED);
448
449 if (held1 != held2) {
450 GLOCK_BUG_ON(gl, __lockref_is_dead(&gl->gl_lockref));
451 if (held2)
452 gl->gl_lockref.count++;
453 else
454 gl->gl_lockref.count--;
455 }
456 if (held1 && held2 && list_empty(&gl->gl_holders))
457 clear_bit(GLF_QUEUED, &gl->gl_flags);
458
459 if (new_state != gl->gl_target)
460
461 gl->gl_hold_time = max(gl->gl_hold_time - GL_GLOCK_HOLD_DECR,
462 GL_GLOCK_MIN_HOLD);
463 gl->gl_state = new_state;
464 gl->gl_tchange = jiffies;
465}
466
467static void gfs2_set_demote(struct gfs2_glock *gl)
468{
469 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
470
471 set_bit(GLF_DEMOTE, &gl->gl_flags);
472 smp_mb();
473 wake_up(&sdp->sd_async_glock_wait);
474}
475
476static void gfs2_demote_wake(struct gfs2_glock *gl)
477{
478 gl->gl_demote_state = LM_ST_EXCLUSIVE;
479 clear_bit(GLF_DEMOTE, &gl->gl_flags);
480 smp_mb__after_atomic();
481 wake_up_bit(&gl->gl_flags, GLF_DEMOTE);
482}
483
484
485
486
487
488
489
490
491static void finish_xmote(struct gfs2_glock *gl, unsigned int ret)
492{
493 const struct gfs2_glock_operations *glops = gl->gl_ops;
494 struct gfs2_holder *gh;
495 unsigned state = ret & LM_OUT_ST_MASK;
496 int rv;
497
498 spin_lock(&gl->gl_lockref.lock);
499 trace_gfs2_glock_state_change(gl, state);
500 state_change(gl, state);
501 gh = find_first_waiter(gl);
502
503
504 if (test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) &&
505 state != LM_ST_UNLOCKED && gl->gl_demote_state == LM_ST_UNLOCKED)
506 gl->gl_target = LM_ST_UNLOCKED;
507
508
509 if (unlikely(state != gl->gl_target)) {
510 if (gh && !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags)) {
511
512 if (ret & LM_OUT_CANCELED) {
513 if ((gh->gh_flags & LM_FLAG_PRIORITY) == 0)
514 list_move_tail(&gh->gh_list, &gl->gl_holders);
515 gh = find_first_waiter(gl);
516 gl->gl_target = gh->gh_state;
517 goto retry;
518 }
519
520 if ((ret & LM_OUT_ERROR) ||
521 (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
522 gl->gl_target = gl->gl_state;
523 do_error(gl, ret);
524 goto out;
525 }
526 }
527 switch(state) {
528
529 case LM_ST_UNLOCKED:
530retry:
531 do_xmote(gl, gh, gl->gl_target);
532 break;
533
534 case LM_ST_SHARED:
535 case LM_ST_DEFERRED:
536 do_xmote(gl, gh, LM_ST_UNLOCKED);
537 break;
538 default:
539 fs_err(gl->gl_name.ln_sbd, "wanted %u got %u\n",
540 gl->gl_target, state);
541 GLOCK_BUG_ON(gl, 1);
542 }
543 spin_unlock(&gl->gl_lockref.lock);
544 return;
545 }
546
547
548 if (test_and_clear_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags))
549 gfs2_demote_wake(gl);
550 if (state != LM_ST_UNLOCKED) {
551 if (glops->go_xmote_bh) {
552 spin_unlock(&gl->gl_lockref.lock);
553 rv = glops->go_xmote_bh(gl, gh);
554 spin_lock(&gl->gl_lockref.lock);
555 if (rv) {
556 do_error(gl, rv);
557 goto out;
558 }
559 }
560 rv = do_promote(gl);
561 if (rv == 2)
562 goto out_locked;
563 }
564out:
565 clear_bit(GLF_LOCK, &gl->gl_flags);
566out_locked:
567 spin_unlock(&gl->gl_lockref.lock);
568}
569
570
571
572
573
574
575
576
577
578static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned int target)
579__releases(&gl->gl_lockref.lock)
580__acquires(&gl->gl_lockref.lock)
581{
582 const struct gfs2_glock_operations *glops = gl->gl_ops;
583 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
584 unsigned int lck_flags = (unsigned int)(gh ? gh->gh_flags : 0);
585 int ret;
586
587 if (target != LM_ST_UNLOCKED && glock_blocked_by_withdraw(gl) &&
588 gh && !(gh->gh_flags & LM_FLAG_NOEXP))
589 return;
590 lck_flags &= (LM_FLAG_TRY | LM_FLAG_TRY_1CB | LM_FLAG_NOEXP |
591 LM_FLAG_PRIORITY);
592 GLOCK_BUG_ON(gl, gl->gl_state == target);
593 GLOCK_BUG_ON(gl, gl->gl_state == gl->gl_target);
594 if ((target == LM_ST_UNLOCKED || target == LM_ST_DEFERRED) &&
595 glops->go_inval) {
596
597
598
599
600
601 if (test_and_set_bit(GLF_INVALIDATE_IN_PROGRESS,
602 &gl->gl_flags))
603 return;
604 do_error(gl, 0);
605 }
606 gl->gl_req = target;
607 set_bit(GLF_BLOCKING, &gl->gl_flags);
608 if ((gl->gl_req == LM_ST_UNLOCKED) ||
609 (gl->gl_state == LM_ST_EXCLUSIVE) ||
610 (lck_flags & (LM_FLAG_TRY|LM_FLAG_TRY_1CB)))
611 clear_bit(GLF_BLOCKING, &gl->gl_flags);
612 spin_unlock(&gl->gl_lockref.lock);
613 if (glops->go_sync) {
614 ret = glops->go_sync(gl);
615
616
617
618
619 if (ret) {
620 if (cmpxchg(&sdp->sd_log_error, 0, ret)) {
621 fs_err(sdp, "Error %d syncing glock \n", ret);
622 gfs2_dump_glock(NULL, gl, true);
623 }
624 goto skip_inval;
625 }
626 }
627 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags)) {
628
629
630
631
632
633
634
635 if ((atomic_read(&gl->gl_ail_count) != 0) &&
636 (!cmpxchg(&sdp->sd_log_error, 0, -EIO))) {
637 gfs2_glock_assert_warn(gl,
638 !atomic_read(&gl->gl_ail_count));
639 gfs2_dump_glock(NULL, gl, true);
640 }
641 glops->go_inval(gl, target == LM_ST_DEFERRED ? 0 : DIO_METADATA);
642 clear_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags);
643 }
644
645skip_inval:
646 gfs2_glock_hold(gl);
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663 if (unlikely(sdp->sd_log_error && !gfs2_withdrawn(sdp)))
664 gfs2_withdraw_delayed(sdp);
665 if (glock_blocked_by_withdraw(gl)) {
666 if (target != LM_ST_UNLOCKED ||
667 test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags)) {
668 gfs2_glock_queue_work(gl, GL_GLOCK_DFT_HOLD);
669 goto out;
670 }
671 }
672
673 if (sdp->sd_lockstruct.ls_ops->lm_lock) {
674
675 ret = sdp->sd_lockstruct.ls_ops->lm_lock(gl, target, lck_flags);
676 if (ret == -EINVAL && gl->gl_target == LM_ST_UNLOCKED &&
677 target == LM_ST_UNLOCKED &&
678 test_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags)) {
679 finish_xmote(gl, target);
680 gfs2_glock_queue_work(gl, 0);
681 } else if (ret) {
682 fs_err(sdp, "lm_lock ret %d\n", ret);
683 GLOCK_BUG_ON(gl, !gfs2_withdrawn(sdp));
684 }
685 } else {
686 finish_xmote(gl, target);
687 gfs2_glock_queue_work(gl, 0);
688 }
689out:
690 spin_lock(&gl->gl_lockref.lock);
691}
692
693
694
695
696
697
698static inline struct gfs2_holder *find_first_holder(const struct gfs2_glock *gl)
699{
700 struct gfs2_holder *gh;
701
702 if (!list_empty(&gl->gl_holders)) {
703 gh = list_first_entry(&gl->gl_holders, struct gfs2_holder, gh_list);
704 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
705 return gh;
706 }
707 return NULL;
708}
709
710
711
712
713
714
715
716
717static void run_queue(struct gfs2_glock *gl, const int nonblock)
718__releases(&gl->gl_lockref.lock)
719__acquires(&gl->gl_lockref.lock)
720{
721 struct gfs2_holder *gh = NULL;
722 int ret;
723
724 if (test_and_set_bit(GLF_LOCK, &gl->gl_flags))
725 return;
726
727 GLOCK_BUG_ON(gl, test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags));
728
729 if (test_bit(GLF_DEMOTE, &gl->gl_flags) &&
730 gl->gl_demote_state != gl->gl_state) {
731 if (find_first_holder(gl))
732 goto out_unlock;
733 if (nonblock)
734 goto out_sched;
735 set_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags);
736 GLOCK_BUG_ON(gl, gl->gl_demote_state == LM_ST_EXCLUSIVE);
737 gl->gl_target = gl->gl_demote_state;
738 } else {
739 if (test_bit(GLF_DEMOTE, &gl->gl_flags))
740 gfs2_demote_wake(gl);
741 ret = do_promote(gl);
742 if (ret == 0)
743 goto out_unlock;
744 if (ret == 2)
745 goto out;
746 gh = find_first_waiter(gl);
747 gl->gl_target = gh->gh_state;
748 if (!(gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)))
749 do_error(gl, 0);
750 }
751 do_xmote(gl, gh, gl->gl_target);
752out:
753 return;
754
755out_sched:
756 clear_bit(GLF_LOCK, &gl->gl_flags);
757 smp_mb__after_atomic();
758 gl->gl_lockref.count++;
759 __gfs2_glock_queue_work(gl, 0);
760 return;
761
762out_unlock:
763 clear_bit(GLF_LOCK, &gl->gl_flags);
764 smp_mb__after_atomic();
765 return;
766}
767
768void gfs2_inode_remember_delete(struct gfs2_glock *gl, u64 generation)
769{
770 struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
771
772 if (ri->ri_magic == 0)
773 ri->ri_magic = cpu_to_be32(GFS2_MAGIC);
774 if (ri->ri_magic == cpu_to_be32(GFS2_MAGIC))
775 ri->ri_generation_deleted = cpu_to_be64(generation);
776}
777
778bool gfs2_inode_already_deleted(struct gfs2_glock *gl, u64 generation)
779{
780 struct gfs2_inode_lvb *ri = (void *)gl->gl_lksb.sb_lvbptr;
781
782 if (ri->ri_magic != cpu_to_be32(GFS2_MAGIC))
783 return false;
784 return generation <= be64_to_cpu(ri->ri_generation_deleted);
785}
786
787static void gfs2_glock_poke(struct gfs2_glock *gl)
788{
789 int flags = LM_FLAG_TRY_1CB | LM_FLAG_ANY | GL_SKIP;
790 struct gfs2_holder gh;
791 int error;
792
793 error = gfs2_glock_nq_init(gl, LM_ST_SHARED, flags, &gh);
794 if (!error)
795 gfs2_glock_dq(&gh);
796}
797
798static bool gfs2_try_evict(struct gfs2_glock *gl)
799{
800 struct gfs2_inode *ip;
801 bool evicted = false;
802
803
804
805
806
807
808
809
810
811
812
813 spin_lock(&gl->gl_lockref.lock);
814 ip = gl->gl_object;
815 if (ip && !igrab(&ip->i_inode))
816 ip = NULL;
817 spin_unlock(&gl->gl_lockref.lock);
818 if (ip) {
819 struct gfs2_glock *inode_gl = NULL;
820
821 gl->gl_no_formal_ino = ip->i_no_formal_ino;
822 set_bit(GIF_DEFERRED_DELETE, &ip->i_flags);
823 d_prune_aliases(&ip->i_inode);
824 iput(&ip->i_inode);
825
826
827 spin_lock(&gl->gl_lockref.lock);
828 ip = gl->gl_object;
829 if (ip) {
830 inode_gl = ip->i_gl;
831 lockref_get(&inode_gl->gl_lockref);
832 clear_bit(GIF_DEFERRED_DELETE, &ip->i_flags);
833 }
834 spin_unlock(&gl->gl_lockref.lock);
835 if (inode_gl) {
836 gfs2_glock_poke(inode_gl);
837 gfs2_glock_put(inode_gl);
838 }
839 evicted = !ip;
840 }
841 return evicted;
842}
843
844static void delete_work_func(struct work_struct *work)
845{
846 struct delayed_work *dwork = to_delayed_work(work);
847 struct gfs2_glock *gl = container_of(dwork, struct gfs2_glock, gl_delete);
848 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
849 struct inode *inode;
850 u64 no_addr = gl->gl_name.ln_number;
851
852 spin_lock(&gl->gl_lockref.lock);
853 clear_bit(GLF_PENDING_DELETE, &gl->gl_flags);
854 spin_unlock(&gl->gl_lockref.lock);
855
856
857
858
859 if (test_bit(GLF_INODE_CREATING, &gl->gl_flags))
860 goto out;
861
862 if (test_bit(GLF_DEMOTE, &gl->gl_flags)) {
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880 if (gfs2_try_evict(gl)) {
881 if (gfs2_queue_delete_work(gl, 5 * HZ))
882 return;
883 }
884 goto out;
885 }
886
887 inode = gfs2_lookup_by_inum(sdp, no_addr, gl->gl_no_formal_ino,
888 GFS2_BLKST_UNLINKED);
889 if (!IS_ERR_OR_NULL(inode)) {
890 d_prune_aliases(inode);
891 iput(inode);
892 }
893out:
894 gfs2_glock_put(gl);
895}
896
897static void glock_work_func(struct work_struct *work)
898{
899 unsigned long delay = 0;
900 struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work.work);
901 unsigned int drop_refs = 1;
902
903 if (test_and_clear_bit(GLF_REPLY_PENDING, &gl->gl_flags)) {
904 finish_xmote(gl, gl->gl_reply);
905 drop_refs++;
906 }
907 spin_lock(&gl->gl_lockref.lock);
908 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
909 gl->gl_state != LM_ST_UNLOCKED &&
910 gl->gl_demote_state != LM_ST_EXCLUSIVE) {
911 unsigned long holdtime, now = jiffies;
912
913 holdtime = gl->gl_tchange + gl->gl_hold_time;
914 if (time_before(now, holdtime))
915 delay = holdtime - now;
916
917 if (!delay) {
918 clear_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
919 gfs2_set_demote(gl);
920 }
921 }
922 run_queue(gl, 0);
923 if (delay) {
924
925 drop_refs--;
926 if (gl->gl_name.ln_type != LM_TYPE_INODE)
927 delay = 0;
928 __gfs2_glock_queue_work(gl, delay);
929 }
930
931
932
933
934
935
936 gl->gl_lockref.count -= drop_refs;
937 if (!gl->gl_lockref.count) {
938 __gfs2_glock_put(gl);
939 return;
940 }
941 spin_unlock(&gl->gl_lockref.lock);
942}
943
944static struct gfs2_glock *find_insert_glock(struct lm_lockname *name,
945 struct gfs2_glock *new)
946{
947 struct wait_glock_queue wait;
948 wait_queue_head_t *wq = glock_waitqueue(name);
949 struct gfs2_glock *gl;
950
951 wait.name = name;
952 init_wait(&wait.wait);
953 wait.wait.func = glock_wake_function;
954
955again:
956 prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE);
957 rcu_read_lock();
958 if (new) {
959 gl = rhashtable_lookup_get_insert_fast(&gl_hash_table,
960 &new->gl_node, ht_parms);
961 if (IS_ERR(gl))
962 goto out;
963 } else {
964 gl = rhashtable_lookup_fast(&gl_hash_table,
965 name, ht_parms);
966 }
967 if (gl && !lockref_get_not_dead(&gl->gl_lockref)) {
968 rcu_read_unlock();
969 schedule();
970 goto again;
971 }
972out:
973 rcu_read_unlock();
974 finish_wait(wq, &wait.wait);
975 return gl;
976}
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
992 const struct gfs2_glock_operations *glops, int create,
993 struct gfs2_glock **glp)
994{
995 struct super_block *s = sdp->sd_vfs;
996 struct lm_lockname name = { .ln_number = number,
997 .ln_type = glops->go_type,
998 .ln_sbd = sdp };
999 struct gfs2_glock *gl, *tmp;
1000 struct address_space *mapping;
1001 struct kmem_cache *cachep;
1002 int ret = 0;
1003
1004 gl = find_insert_glock(&name, NULL);
1005 if (gl) {
1006 *glp = gl;
1007 return 0;
1008 }
1009 if (!create)
1010 return -ENOENT;
1011
1012 if (glops->go_flags & GLOF_ASPACE)
1013 cachep = gfs2_glock_aspace_cachep;
1014 else
1015 cachep = gfs2_glock_cachep;
1016 gl = kmem_cache_alloc(cachep, GFP_NOFS);
1017 if (!gl)
1018 return -ENOMEM;
1019
1020 memset(&gl->gl_lksb, 0, sizeof(struct dlm_lksb));
1021
1022 if (glops->go_flags & GLOF_LVB) {
1023 gl->gl_lksb.sb_lvbptr = kzalloc(GDLM_LVB_SIZE, GFP_NOFS);
1024 if (!gl->gl_lksb.sb_lvbptr) {
1025 kmem_cache_free(cachep, gl);
1026 return -ENOMEM;
1027 }
1028 }
1029
1030 atomic_inc(&sdp->sd_glock_disposal);
1031 gl->gl_node.next = NULL;
1032 gl->gl_flags = 0;
1033 gl->gl_name = name;
1034 gl->gl_lockref.count = 1;
1035 gl->gl_state = LM_ST_UNLOCKED;
1036 gl->gl_target = LM_ST_UNLOCKED;
1037 gl->gl_demote_state = LM_ST_EXCLUSIVE;
1038 gl->gl_ops = glops;
1039 gl->gl_dstamp = 0;
1040 preempt_disable();
1041
1042 gl->gl_stats = this_cpu_ptr(sdp->sd_lkstats)->lkstats[glops->go_type];
1043 preempt_enable();
1044 gl->gl_stats.stats[GFS2_LKS_DCOUNT] = 0;
1045 gl->gl_stats.stats[GFS2_LKS_QCOUNT] = 0;
1046 gl->gl_tchange = jiffies;
1047 gl->gl_object = NULL;
1048 gl->gl_hold_time = GL_GLOCK_DFT_HOLD;
1049 INIT_DELAYED_WORK(&gl->gl_work, glock_work_func);
1050 INIT_DELAYED_WORK(&gl->gl_delete, delete_work_func);
1051
1052 mapping = gfs2_glock2aspace(gl);
1053 if (mapping) {
1054 mapping->a_ops = &gfs2_meta_aops;
1055 mapping->host = s->s_bdev->bd_inode;
1056 mapping->flags = 0;
1057 mapping_set_gfp_mask(mapping, GFP_NOFS);
1058 mapping->private_data = NULL;
1059 mapping->writeback_index = 0;
1060 }
1061
1062 tmp = find_insert_glock(&name, gl);
1063 if (!tmp) {
1064 *glp = gl;
1065 goto out;
1066 }
1067 if (IS_ERR(tmp)) {
1068 ret = PTR_ERR(tmp);
1069 goto out_free;
1070 }
1071 *glp = tmp;
1072
1073out_free:
1074 kfree(gl->gl_lksb.sb_lvbptr);
1075 kmem_cache_free(cachep, gl);
1076 atomic_dec(&sdp->sd_glock_disposal);
1077
1078out:
1079 return ret;
1080}
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091void gfs2_holder_init(struct gfs2_glock *gl, unsigned int state, u16 flags,
1092 struct gfs2_holder *gh)
1093{
1094 INIT_LIST_HEAD(&gh->gh_list);
1095 gh->gh_gl = gl;
1096 gh->gh_ip = _RET_IP_;
1097 gh->gh_owner_pid = get_pid(task_pid(current));
1098 gh->gh_state = state;
1099 gh->gh_flags = flags;
1100 gh->gh_error = 0;
1101 gh->gh_iflags = 0;
1102 gfs2_glock_hold(gl);
1103}
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115void gfs2_holder_reinit(unsigned int state, u16 flags, struct gfs2_holder *gh)
1116{
1117 gh->gh_state = state;
1118 gh->gh_flags = flags;
1119 gh->gh_iflags = 0;
1120 gh->gh_ip = _RET_IP_;
1121 put_pid(gh->gh_owner_pid);
1122 gh->gh_owner_pid = get_pid(task_pid(current));
1123}
1124
1125
1126
1127
1128
1129
1130
1131void gfs2_holder_uninit(struct gfs2_holder *gh)
1132{
1133 put_pid(gh->gh_owner_pid);
1134 gfs2_glock_put(gh->gh_gl);
1135 gfs2_holder_mark_uninitialized(gh);
1136 gh->gh_ip = 0;
1137}
1138
1139static void gfs2_glock_update_hold_time(struct gfs2_glock *gl,
1140 unsigned long start_time)
1141{
1142
1143 if (time_after(jiffies, start_time + HZ)) {
1144
1145 gl->gl_hold_time = min(gl->gl_hold_time + GL_GLOCK_HOLD_INCR,
1146 GL_GLOCK_MAX_HOLD);
1147 }
1148}
1149
1150
1151
1152
1153
1154
1155
1156
1157int gfs2_glock_wait(struct gfs2_holder *gh)
1158{
1159 unsigned long start_time = jiffies;
1160
1161 might_sleep();
1162 wait_on_bit(&gh->gh_iflags, HIF_WAIT, TASK_UNINTERRUPTIBLE);
1163 gfs2_glock_update_hold_time(gh->gh_gl, start_time);
1164 return gh->gh_error;
1165}
1166
1167static int glocks_pending(unsigned int num_gh, struct gfs2_holder *ghs)
1168{
1169 int i;
1170
1171 for (i = 0; i < num_gh; i++)
1172 if (test_bit(HIF_WAIT, &ghs[i].gh_iflags))
1173 return 1;
1174 return 0;
1175}
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187int gfs2_glock_async_wait(unsigned int num_gh, struct gfs2_holder *ghs)
1188{
1189 struct gfs2_sbd *sdp = ghs[0].gh_gl->gl_name.ln_sbd;
1190 int i, ret = 0, timeout = 0;
1191 unsigned long start_time = jiffies;
1192 bool keep_waiting;
1193
1194 might_sleep();
1195
1196
1197
1198
1199 for (i = 0; i < num_gh; i++)
1200 timeout += ghs[i].gh_gl->gl_hold_time << 1;
1201
1202wait_for_dlm:
1203 if (!wait_event_timeout(sdp->sd_async_glock_wait,
1204 !glocks_pending(num_gh, ghs), timeout))
1205 ret = -ESTALE;
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221 keep_waiting = false;
1222 for (i = 0; i < num_gh; i++) {
1223
1224 if (!gfs2_holder_queued(&ghs[i]))
1225 continue;
1226
1227 if (test_bit(HIF_WAIT, &ghs[i].gh_iflags)) {
1228 keep_waiting = true;
1229 continue;
1230 }
1231
1232 if (test_bit(HIF_HOLDER, &ghs[i].gh_iflags)) {
1233 if (ret == -ESTALE)
1234 gfs2_glock_dq(&ghs[i]);
1235 else
1236 gfs2_glock_update_hold_time(ghs[i].gh_gl,
1237 start_time);
1238 }
1239 if (!ret)
1240 ret = ghs[i].gh_error;
1241 }
1242
1243 if (keep_waiting)
1244 goto wait_for_dlm;
1245
1246
1247
1248
1249 return ret;
1250}
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261static void handle_callback(struct gfs2_glock *gl, unsigned int state,
1262 unsigned long delay, bool remote)
1263{
1264 if (delay)
1265 set_bit(GLF_PENDING_DEMOTE, &gl->gl_flags);
1266 else
1267 gfs2_set_demote(gl);
1268 if (gl->gl_demote_state == LM_ST_EXCLUSIVE) {
1269 gl->gl_demote_state = state;
1270 gl->gl_demote_time = jiffies;
1271 } else if (gl->gl_demote_state != LM_ST_UNLOCKED &&
1272 gl->gl_demote_state != state) {
1273 gl->gl_demote_state = LM_ST_UNLOCKED;
1274 }
1275 if (gl->gl_ops->go_callback)
1276 gl->gl_ops->go_callback(gl, remote);
1277 trace_gfs2_demote_rq(gl, remote);
1278}
1279
1280void gfs2_print_dbg(struct seq_file *seq, const char *fmt, ...)
1281{
1282 struct va_format vaf;
1283 va_list args;
1284
1285 va_start(args, fmt);
1286
1287 if (seq) {
1288 seq_vprintf(seq, fmt, args);
1289 } else {
1290 vaf.fmt = fmt;
1291 vaf.va = &args;
1292
1293 pr_err("%pV", &vaf);
1294 }
1295
1296 va_end(args);
1297}
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309static inline void add_to_queue(struct gfs2_holder *gh)
1310__releases(&gl->gl_lockref.lock)
1311__acquires(&gl->gl_lockref.lock)
1312{
1313 struct gfs2_glock *gl = gh->gh_gl;
1314 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1315 struct list_head *insert_pt = NULL;
1316 struct gfs2_holder *gh2;
1317 int try_futile = 0;
1318
1319 GLOCK_BUG_ON(gl, gh->gh_owner_pid == NULL);
1320 if (test_and_set_bit(HIF_WAIT, &gh->gh_iflags))
1321 GLOCK_BUG_ON(gl, true);
1322
1323 if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) {
1324 if (test_bit(GLF_LOCK, &gl->gl_flags))
1325 try_futile = !may_grant(gl, gh);
1326 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, &gl->gl_flags))
1327 goto fail;
1328 }
1329
1330 list_for_each_entry(gh2, &gl->gl_holders, gh_list) {
1331 if (unlikely(gh2->gh_owner_pid == gh->gh_owner_pid &&
1332 (gh->gh_gl->gl_ops->go_type != LM_TYPE_FLOCK)))
1333 goto trap_recursive;
1334 if (try_futile &&
1335 !(gh2->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))) {
1336fail:
1337 gh->gh_error = GLR_TRYFAILED;
1338 gfs2_holder_wake(gh);
1339 return;
1340 }
1341 if (test_bit(HIF_HOLDER, &gh2->gh_iflags))
1342 continue;
1343 if (unlikely((gh->gh_flags & LM_FLAG_PRIORITY) && !insert_pt))
1344 insert_pt = &gh2->gh_list;
1345 }
1346 set_bit(GLF_QUEUED, &gl->gl_flags);
1347 trace_gfs2_glock_queue(gh, 1);
1348 gfs2_glstats_inc(gl, GFS2_LKS_QCOUNT);
1349 gfs2_sbstats_inc(gl, GFS2_LKS_QCOUNT);
1350 if (likely(insert_pt == NULL)) {
1351 list_add_tail(&gh->gh_list, &gl->gl_holders);
1352 if (unlikely(gh->gh_flags & LM_FLAG_PRIORITY))
1353 goto do_cancel;
1354 return;
1355 }
1356 list_add_tail(&gh->gh_list, insert_pt);
1357do_cancel:
1358 gh = list_first_entry(&gl->gl_holders, struct gfs2_holder, gh_list);
1359 if (!(gh->gh_flags & LM_FLAG_PRIORITY)) {
1360 spin_unlock(&gl->gl_lockref.lock);
1361 if (sdp->sd_lockstruct.ls_ops->lm_cancel)
1362 sdp->sd_lockstruct.ls_ops->lm_cancel(gl);
1363 spin_lock(&gl->gl_lockref.lock);
1364 }
1365 return;
1366
1367trap_recursive:
1368 fs_err(sdp, "original: %pSR\n", (void *)gh2->gh_ip);
1369 fs_err(sdp, "pid: %d\n", pid_nr(gh2->gh_owner_pid));
1370 fs_err(sdp, "lock type: %d req lock state : %d\n",
1371 gh2->gh_gl->gl_name.ln_type, gh2->gh_state);
1372 fs_err(sdp, "new: %pSR\n", (void *)gh->gh_ip);
1373 fs_err(sdp, "pid: %d\n", pid_nr(gh->gh_owner_pid));
1374 fs_err(sdp, "lock type: %d req lock state : %d\n",
1375 gh->gh_gl->gl_name.ln_type, gh->gh_state);
1376 gfs2_dump_glock(NULL, gl, true);
1377 BUG();
1378}
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389int gfs2_glock_nq(struct gfs2_holder *gh)
1390{
1391 struct gfs2_glock *gl = gh->gh_gl;
1392 int error = 0;
1393
1394 if (glock_blocked_by_withdraw(gl) && !(gh->gh_flags & LM_FLAG_NOEXP))
1395 return -EIO;
1396
1397 if (test_bit(GLF_LRU, &gl->gl_flags))
1398 gfs2_glock_remove_from_lru(gl);
1399
1400 spin_lock(&gl->gl_lockref.lock);
1401 add_to_queue(gh);
1402 if (unlikely((LM_FLAG_NOEXP & gh->gh_flags) &&
1403 test_and_clear_bit(GLF_FROZEN, &gl->gl_flags))) {
1404 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1405 gl->gl_lockref.count++;
1406 __gfs2_glock_queue_work(gl, 0);
1407 }
1408 run_queue(gl, 1);
1409 spin_unlock(&gl->gl_lockref.lock);
1410
1411 if (!(gh->gh_flags & GL_ASYNC))
1412 error = gfs2_glock_wait(gh);
1413
1414 return error;
1415}
1416
1417
1418
1419
1420
1421
1422
1423
1424int gfs2_glock_poll(struct gfs2_holder *gh)
1425{
1426 return test_bit(HIF_WAIT, &gh->gh_iflags) ? 0 : 1;
1427}
1428
1429
1430
1431
1432
1433
1434
1435void gfs2_glock_dq(struct gfs2_holder *gh)
1436{
1437 struct gfs2_glock *gl = gh->gh_gl;
1438 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1439 unsigned delay = 0;
1440 int fast_path = 0;
1441
1442 spin_lock(&gl->gl_lockref.lock);
1443
1444
1445
1446
1447
1448
1449
1450 if (test_bit(SDF_WITHDRAW_RECOVERY, &sdp->sd_flags) &&
1451 glock_blocked_by_withdraw(gl) &&
1452 gh->gh_gl != sdp->sd_jinode_gl) {
1453 sdp->sd_glock_dqs_held++;
1454 might_sleep();
1455 wait_on_bit(&sdp->sd_flags, SDF_WITHDRAW_RECOVERY,
1456 TASK_UNINTERRUPTIBLE);
1457 }
1458 if (gh->gh_flags & GL_NOCACHE)
1459 handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1460
1461 list_del_init(&gh->gh_list);
1462 clear_bit(HIF_HOLDER, &gh->gh_iflags);
1463 if (find_first_holder(gl) == NULL) {
1464 if (list_empty(&gl->gl_holders) &&
1465 !test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1466 !test_bit(GLF_DEMOTE, &gl->gl_flags))
1467 fast_path = 1;
1468 }
1469 if (!test_bit(GLF_LFLUSH, &gl->gl_flags) && demote_ok(gl))
1470 gfs2_glock_add_to_lru(gl);
1471
1472 trace_gfs2_glock_queue(gh, 0);
1473 if (unlikely(!fast_path)) {
1474 gl->gl_lockref.count++;
1475 if (test_bit(GLF_PENDING_DEMOTE, &gl->gl_flags) &&
1476 !test_bit(GLF_DEMOTE, &gl->gl_flags) &&
1477 gl->gl_name.ln_type == LM_TYPE_INODE)
1478 delay = gl->gl_hold_time;
1479 __gfs2_glock_queue_work(gl, delay);
1480 }
1481 spin_unlock(&gl->gl_lockref.lock);
1482}
1483
1484void gfs2_glock_dq_wait(struct gfs2_holder *gh)
1485{
1486 struct gfs2_glock *gl = gh->gh_gl;
1487 gfs2_glock_dq(gh);
1488 might_sleep();
1489 wait_on_bit(&gl->gl_flags, GLF_DEMOTE, TASK_UNINTERRUPTIBLE);
1490}
1491
1492
1493
1494
1495
1496
1497
1498void gfs2_glock_dq_uninit(struct gfs2_holder *gh)
1499{
1500 gfs2_glock_dq(gh);
1501 gfs2_holder_uninit(gh);
1502}
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516int gfs2_glock_nq_num(struct gfs2_sbd *sdp, u64 number,
1517 const struct gfs2_glock_operations *glops,
1518 unsigned int state, u16 flags, struct gfs2_holder *gh)
1519{
1520 struct gfs2_glock *gl;
1521 int error;
1522
1523 error = gfs2_glock_get(sdp, number, glops, CREATE, &gl);
1524 if (!error) {
1525 error = gfs2_glock_nq_init(gl, state, flags, gh);
1526 gfs2_glock_put(gl);
1527 }
1528
1529 return error;
1530}
1531
1532
1533
1534
1535
1536
1537
1538
1539static int glock_compare(const void *arg_a, const void *arg_b)
1540{
1541 const struct gfs2_holder *gh_a = *(const struct gfs2_holder **)arg_a;
1542 const struct gfs2_holder *gh_b = *(const struct gfs2_holder **)arg_b;
1543 const struct lm_lockname *a = &gh_a->gh_gl->gl_name;
1544 const struct lm_lockname *b = &gh_b->gh_gl->gl_name;
1545
1546 if (a->ln_number > b->ln_number)
1547 return 1;
1548 if (a->ln_number < b->ln_number)
1549 return -1;
1550 BUG_ON(gh_a->gh_gl->gl_ops->go_type == gh_b->gh_gl->gl_ops->go_type);
1551 return 0;
1552}
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563static int nq_m_sync(unsigned int num_gh, struct gfs2_holder *ghs,
1564 struct gfs2_holder **p)
1565{
1566 unsigned int x;
1567 int error = 0;
1568
1569 for (x = 0; x < num_gh; x++)
1570 p[x] = &ghs[x];
1571
1572 sort(p, num_gh, sizeof(struct gfs2_holder *), glock_compare, NULL);
1573
1574 for (x = 0; x < num_gh; x++) {
1575 p[x]->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC);
1576
1577 error = gfs2_glock_nq(p[x]);
1578 if (error) {
1579 while (x--)
1580 gfs2_glock_dq(p[x]);
1581 break;
1582 }
1583 }
1584
1585 return error;
1586}
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598int gfs2_glock_nq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1599{
1600 struct gfs2_holder *tmp[4];
1601 struct gfs2_holder **pph = tmp;
1602 int error = 0;
1603
1604 switch(num_gh) {
1605 case 0:
1606 return 0;
1607 case 1:
1608 ghs->gh_flags &= ~(LM_FLAG_TRY | GL_ASYNC);
1609 return gfs2_glock_nq(ghs);
1610 default:
1611 if (num_gh <= 4)
1612 break;
1613 pph = kmalloc_array(num_gh, sizeof(struct gfs2_holder *),
1614 GFP_NOFS);
1615 if (!pph)
1616 return -ENOMEM;
1617 }
1618
1619 error = nq_m_sync(num_gh, ghs, pph);
1620
1621 if (pph != tmp)
1622 kfree(pph);
1623
1624 return error;
1625}
1626
1627
1628
1629
1630
1631
1632
1633
1634void gfs2_glock_dq_m(unsigned int num_gh, struct gfs2_holder *ghs)
1635{
1636 while (num_gh--)
1637 gfs2_glock_dq(&ghs[num_gh]);
1638}
1639
1640void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int state)
1641{
1642 unsigned long delay = 0;
1643 unsigned long holdtime;
1644 unsigned long now = jiffies;
1645
1646 gfs2_glock_hold(gl);
1647 holdtime = gl->gl_tchange + gl->gl_hold_time;
1648 if (test_bit(GLF_QUEUED, &gl->gl_flags) &&
1649 gl->gl_name.ln_type == LM_TYPE_INODE) {
1650 if (time_before(now, holdtime))
1651 delay = holdtime - now;
1652 if (test_bit(GLF_REPLY_PENDING, &gl->gl_flags))
1653 delay = gl->gl_hold_time;
1654 }
1655
1656 spin_lock(&gl->gl_lockref.lock);
1657 handle_callback(gl, state, delay, true);
1658 __gfs2_glock_queue_work(gl, delay);
1659 spin_unlock(&gl->gl_lockref.lock);
1660}
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673static int gfs2_should_freeze(const struct gfs2_glock *gl)
1674{
1675 const struct gfs2_holder *gh;
1676
1677 if (gl->gl_reply & ~LM_OUT_ST_MASK)
1678 return 0;
1679 if (gl->gl_target == LM_ST_UNLOCKED)
1680 return 0;
1681
1682 list_for_each_entry(gh, &gl->gl_holders, gh_list) {
1683 if (test_bit(HIF_HOLDER, &gh->gh_iflags))
1684 continue;
1685 if (LM_FLAG_NOEXP & gh->gh_flags)
1686 return 0;
1687 }
1688
1689 return 1;
1690}
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701void gfs2_glock_complete(struct gfs2_glock *gl, int ret)
1702{
1703 struct lm_lockstruct *ls = &gl->gl_name.ln_sbd->sd_lockstruct;
1704
1705 spin_lock(&gl->gl_lockref.lock);
1706 gl->gl_reply = ret;
1707
1708 if (unlikely(test_bit(DFL_BLOCK_LOCKS, &ls->ls_recover_flags))) {
1709 if (gfs2_should_freeze(gl)) {
1710 set_bit(GLF_FROZEN, &gl->gl_flags);
1711 spin_unlock(&gl->gl_lockref.lock);
1712 return;
1713 }
1714 }
1715
1716 gl->gl_lockref.count++;
1717 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1718 __gfs2_glock_queue_work(gl, 0);
1719 spin_unlock(&gl->gl_lockref.lock);
1720}
1721
1722static int glock_cmp(void *priv, struct list_head *a, struct list_head *b)
1723{
1724 struct gfs2_glock *gla, *glb;
1725
1726 gla = list_entry(a, struct gfs2_glock, gl_lru);
1727 glb = list_entry(b, struct gfs2_glock, gl_lru);
1728
1729 if (gla->gl_name.ln_number > glb->gl_name.ln_number)
1730 return 1;
1731 if (gla->gl_name.ln_number < glb->gl_name.ln_number)
1732 return -1;
1733
1734 return 0;
1735}
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751static void gfs2_dispose_glock_lru(struct list_head *list)
1752__releases(&lru_lock)
1753__acquires(&lru_lock)
1754{
1755 struct gfs2_glock *gl;
1756
1757 list_sort(NULL, list, glock_cmp);
1758
1759 while(!list_empty(list)) {
1760 gl = list_first_entry(list, struct gfs2_glock, gl_lru);
1761 list_del_init(&gl->gl_lru);
1762 if (!spin_trylock(&gl->gl_lockref.lock)) {
1763add_back_to_lru:
1764 list_add(&gl->gl_lru, &lru_list);
1765 set_bit(GLF_LRU, &gl->gl_flags);
1766 atomic_inc(&lru_count);
1767 continue;
1768 }
1769 if (test_and_set_bit(GLF_LOCK, &gl->gl_flags)) {
1770 spin_unlock(&gl->gl_lockref.lock);
1771 goto add_back_to_lru;
1772 }
1773 gl->gl_lockref.count++;
1774 if (demote_ok(gl))
1775 handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1776 WARN_ON(!test_and_clear_bit(GLF_LOCK, &gl->gl_flags));
1777 __gfs2_glock_queue_work(gl, 0);
1778 spin_unlock(&gl->gl_lockref.lock);
1779 cond_resched_lock(&lru_lock);
1780 }
1781}
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792static long gfs2_scan_glock_lru(int nr)
1793{
1794 struct gfs2_glock *gl;
1795 LIST_HEAD(skipped);
1796 LIST_HEAD(dispose);
1797 long freed = 0;
1798
1799 spin_lock(&lru_lock);
1800 while ((nr-- >= 0) && !list_empty(&lru_list)) {
1801 gl = list_first_entry(&lru_list, struct gfs2_glock, gl_lru);
1802
1803
1804 if (!test_bit(GLF_LOCK, &gl->gl_flags)) {
1805 list_move(&gl->gl_lru, &dispose);
1806 atomic_dec(&lru_count);
1807 clear_bit(GLF_LRU, &gl->gl_flags);
1808 freed++;
1809 continue;
1810 }
1811
1812 list_move(&gl->gl_lru, &skipped);
1813 }
1814 list_splice(&skipped, &lru_list);
1815 if (!list_empty(&dispose))
1816 gfs2_dispose_glock_lru(&dispose);
1817 spin_unlock(&lru_lock);
1818
1819 return freed;
1820}
1821
1822static unsigned long gfs2_glock_shrink_scan(struct shrinker *shrink,
1823 struct shrink_control *sc)
1824{
1825 if (!(sc->gfp_mask & __GFP_FS))
1826 return SHRINK_STOP;
1827 return gfs2_scan_glock_lru(sc->nr_to_scan);
1828}
1829
1830static unsigned long gfs2_glock_shrink_count(struct shrinker *shrink,
1831 struct shrink_control *sc)
1832{
1833 return vfs_pressure_ratio(atomic_read(&lru_count));
1834}
1835
1836static struct shrinker glock_shrinker = {
1837 .seeks = DEFAULT_SEEKS,
1838 .count_objects = gfs2_glock_shrink_count,
1839 .scan_objects = gfs2_glock_shrink_scan,
1840};
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853static void glock_hash_walk(glock_examiner examiner, const struct gfs2_sbd *sdp)
1854{
1855 struct gfs2_glock *gl;
1856 struct rhashtable_iter iter;
1857
1858 rhashtable_walk_enter(&gl_hash_table, &iter);
1859
1860 do {
1861 rhashtable_walk_start(&iter);
1862
1863 while ((gl = rhashtable_walk_next(&iter)) && !IS_ERR(gl))
1864 if (gl->gl_name.ln_sbd == sdp &&
1865 lockref_get_not_dead(&gl->gl_lockref))
1866 examiner(gl);
1867
1868 rhashtable_walk_stop(&iter);
1869 } while (cond_resched(), gl == ERR_PTR(-EAGAIN));
1870
1871 rhashtable_walk_exit(&iter);
1872}
1873
1874bool gfs2_queue_delete_work(struct gfs2_glock *gl, unsigned long delay)
1875{
1876 bool queued;
1877
1878 spin_lock(&gl->gl_lockref.lock);
1879 queued = queue_delayed_work(gfs2_delete_workqueue,
1880 &gl->gl_delete, delay);
1881 if (queued)
1882 set_bit(GLF_PENDING_DELETE, &gl->gl_flags);
1883 spin_unlock(&gl->gl_lockref.lock);
1884 return queued;
1885}
1886
1887void gfs2_cancel_delete_work(struct gfs2_glock *gl)
1888{
1889 if (cancel_delayed_work_sync(&gl->gl_delete)) {
1890 clear_bit(GLF_PENDING_DELETE, &gl->gl_flags);
1891 gfs2_glock_put(gl);
1892 }
1893}
1894
1895bool gfs2_delete_work_queued(const struct gfs2_glock *gl)
1896{
1897 return test_bit(GLF_PENDING_DELETE, &gl->gl_flags);
1898}
1899
1900static void flush_delete_work(struct gfs2_glock *gl)
1901{
1902 if (cancel_delayed_work(&gl->gl_delete)) {
1903 queue_delayed_work(gfs2_delete_workqueue,
1904 &gl->gl_delete, 0);
1905 }
1906 gfs2_glock_queue_work(gl, 0);
1907}
1908
1909void gfs2_flush_delete_work(struct gfs2_sbd *sdp)
1910{
1911 glock_hash_walk(flush_delete_work, sdp);
1912 flush_workqueue(gfs2_delete_workqueue);
1913}
1914
1915
1916
1917
1918
1919
1920
1921static void thaw_glock(struct gfs2_glock *gl)
1922{
1923 if (!test_and_clear_bit(GLF_FROZEN, &gl->gl_flags)) {
1924 gfs2_glock_put(gl);
1925 return;
1926 }
1927 set_bit(GLF_REPLY_PENDING, &gl->gl_flags);
1928 gfs2_glock_queue_work(gl, 0);
1929}
1930
1931
1932
1933
1934
1935
1936
1937static void clear_glock(struct gfs2_glock *gl)
1938{
1939 gfs2_glock_remove_from_lru(gl);
1940
1941 spin_lock(&gl->gl_lockref.lock);
1942 if (gl->gl_state != LM_ST_UNLOCKED)
1943 handle_callback(gl, LM_ST_UNLOCKED, 0, false);
1944 __gfs2_glock_queue_work(gl, 0);
1945 spin_unlock(&gl->gl_lockref.lock);
1946}
1947
1948
1949
1950
1951
1952
1953
1954void gfs2_glock_thaw(struct gfs2_sbd *sdp)
1955{
1956 glock_hash_walk(thaw_glock, sdp);
1957}
1958
1959static void dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
1960{
1961 spin_lock(&gl->gl_lockref.lock);
1962 gfs2_dump_glock(seq, gl, fsid);
1963 spin_unlock(&gl->gl_lockref.lock);
1964}
1965
1966static void dump_glock_func(struct gfs2_glock *gl)
1967{
1968 dump_glock(NULL, gl, true);
1969}
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979void gfs2_gl_hash_clear(struct gfs2_sbd *sdp)
1980{
1981 set_bit(SDF_SKIP_DLM_UNLOCK, &sdp->sd_flags);
1982 flush_workqueue(glock_workqueue);
1983 glock_hash_walk(clear_glock, sdp);
1984 flush_workqueue(glock_workqueue);
1985 wait_event_timeout(sdp->sd_glock_wait,
1986 atomic_read(&sdp->sd_glock_disposal) == 0,
1987 HZ * 600);
1988 glock_hash_walk(dump_glock_func, sdp);
1989}
1990
1991void gfs2_glock_finish_truncate(struct gfs2_inode *ip)
1992{
1993 struct gfs2_glock *gl = ip->i_gl;
1994 int ret;
1995
1996 ret = gfs2_truncatei_resume(ip);
1997 gfs2_glock_assert_withdraw(gl, ret == 0);
1998
1999 spin_lock(&gl->gl_lockref.lock);
2000 clear_bit(GLF_LOCK, &gl->gl_flags);
2001 run_queue(gl, 1);
2002 spin_unlock(&gl->gl_lockref.lock);
2003}
2004
2005static const char *state2str(unsigned state)
2006{
2007 switch(state) {
2008 case LM_ST_UNLOCKED:
2009 return "UN";
2010 case LM_ST_SHARED:
2011 return "SH";
2012 case LM_ST_DEFERRED:
2013 return "DF";
2014 case LM_ST_EXCLUSIVE:
2015 return "EX";
2016 }
2017 return "??";
2018}
2019
2020static const char *hflags2str(char *buf, u16 flags, unsigned long iflags)
2021{
2022 char *p = buf;
2023 if (flags & LM_FLAG_TRY)
2024 *p++ = 't';
2025 if (flags & LM_FLAG_TRY_1CB)
2026 *p++ = 'T';
2027 if (flags & LM_FLAG_NOEXP)
2028 *p++ = 'e';
2029 if (flags & LM_FLAG_ANY)
2030 *p++ = 'A';
2031 if (flags & LM_FLAG_PRIORITY)
2032 *p++ = 'p';
2033 if (flags & GL_ASYNC)
2034 *p++ = 'a';
2035 if (flags & GL_EXACT)
2036 *p++ = 'E';
2037 if (flags & GL_NOCACHE)
2038 *p++ = 'c';
2039 if (test_bit(HIF_HOLDER, &iflags))
2040 *p++ = 'H';
2041 if (test_bit(HIF_WAIT, &iflags))
2042 *p++ = 'W';
2043 if (test_bit(HIF_FIRST, &iflags))
2044 *p++ = 'F';
2045 *p = 0;
2046 return buf;
2047}
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057static void dump_holder(struct seq_file *seq, const struct gfs2_holder *gh,
2058 const char *fs_id_buf)
2059{
2060 struct task_struct *gh_owner = NULL;
2061 char flags_buf[32];
2062
2063 rcu_read_lock();
2064 if (gh->gh_owner_pid)
2065 gh_owner = pid_task(gh->gh_owner_pid, PIDTYPE_PID);
2066 gfs2_print_dbg(seq, "%s H: s:%s f:%s e:%d p:%ld [%s] %pS\n",
2067 fs_id_buf, state2str(gh->gh_state),
2068 hflags2str(flags_buf, gh->gh_flags, gh->gh_iflags),
2069 gh->gh_error,
2070 gh->gh_owner_pid ? (long)pid_nr(gh->gh_owner_pid) : -1,
2071 gh_owner ? gh_owner->comm : "(ended)",
2072 (void *)gh->gh_ip);
2073 rcu_read_unlock();
2074}
2075
2076static const char *gflags2str(char *buf, const struct gfs2_glock *gl)
2077{
2078 const unsigned long *gflags = &gl->gl_flags;
2079 char *p = buf;
2080
2081 if (test_bit(GLF_LOCK, gflags))
2082 *p++ = 'l';
2083 if (test_bit(GLF_DEMOTE, gflags))
2084 *p++ = 'D';
2085 if (test_bit(GLF_PENDING_DEMOTE, gflags))
2086 *p++ = 'd';
2087 if (test_bit(GLF_DEMOTE_IN_PROGRESS, gflags))
2088 *p++ = 'p';
2089 if (test_bit(GLF_DIRTY, gflags))
2090 *p++ = 'y';
2091 if (test_bit(GLF_LFLUSH, gflags))
2092 *p++ = 'f';
2093 if (test_bit(GLF_INVALIDATE_IN_PROGRESS, gflags))
2094 *p++ = 'i';
2095 if (test_bit(GLF_REPLY_PENDING, gflags))
2096 *p++ = 'r';
2097 if (test_bit(GLF_INITIAL, gflags))
2098 *p++ = 'I';
2099 if (test_bit(GLF_FROZEN, gflags))
2100 *p++ = 'F';
2101 if (test_bit(GLF_QUEUED, gflags))
2102 *p++ = 'q';
2103 if (test_bit(GLF_LRU, gflags))
2104 *p++ = 'L';
2105 if (gl->gl_object)
2106 *p++ = 'o';
2107 if (test_bit(GLF_BLOCKING, gflags))
2108 *p++ = 'b';
2109 *p = 0;
2110 return buf;
2111}
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131void gfs2_dump_glock(struct seq_file *seq, struct gfs2_glock *gl, bool fsid)
2132{
2133 const struct gfs2_glock_operations *glops = gl->gl_ops;
2134 unsigned long long dtime;
2135 const struct gfs2_holder *gh;
2136 char gflags_buf[32];
2137 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
2138 char fs_id_buf[sizeof(sdp->sd_fsname) + 7];
2139 unsigned long nrpages = 0;
2140
2141 if (gl->gl_ops->go_flags & GLOF_ASPACE) {
2142 struct address_space *mapping = gfs2_glock2aspace(gl);
2143
2144 nrpages = mapping->nrpages;
2145 }
2146 memset(fs_id_buf, 0, sizeof(fs_id_buf));
2147 if (fsid && sdp)
2148 sprintf(fs_id_buf, "fsid=%s: ", sdp->sd_fsname);
2149 dtime = jiffies - gl->gl_demote_time;
2150 dtime *= 1000000/HZ;
2151 if (!test_bit(GLF_DEMOTE, &gl->gl_flags))
2152 dtime = 0;
2153 gfs2_print_dbg(seq, "%sG: s:%s n:%u/%llx f:%s t:%s d:%s/%llu a:%d "
2154 "v:%d r:%d m:%ld p:%lu\n",
2155 fs_id_buf, state2str(gl->gl_state),
2156 gl->gl_name.ln_type,
2157 (unsigned long long)gl->gl_name.ln_number,
2158 gflags2str(gflags_buf, gl),
2159 state2str(gl->gl_target),
2160 state2str(gl->gl_demote_state), dtime,
2161 atomic_read(&gl->gl_ail_count),
2162 atomic_read(&gl->gl_revokes),
2163 (int)gl->gl_lockref.count, gl->gl_hold_time, nrpages);
2164
2165 list_for_each_entry(gh, &gl->gl_holders, gh_list)
2166 dump_holder(seq, gh, fs_id_buf);
2167
2168 if (gl->gl_state != LM_ST_UNLOCKED && glops->go_dump)
2169 glops->go_dump(seq, gl, fs_id_buf);
2170}
2171
2172static int gfs2_glstats_seq_show(struct seq_file *seq, void *iter_ptr)
2173{
2174 struct gfs2_glock *gl = iter_ptr;
2175
2176 seq_printf(seq, "G: n:%u/%llx rtt:%llu/%llu rttb:%llu/%llu irt:%llu/%llu dcnt: %llu qcnt: %llu\n",
2177 gl->gl_name.ln_type,
2178 (unsigned long long)gl->gl_name.ln_number,
2179 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTT],
2180 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVAR],
2181 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTB],
2182 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SRTTVARB],
2183 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRT],
2184 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_SIRTVAR],
2185 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_DCOUNT],
2186 (unsigned long long)gl->gl_stats.stats[GFS2_LKS_QCOUNT]);
2187 return 0;
2188}
2189
2190static const char *gfs2_gltype[] = {
2191 "type",
2192 "reserved",
2193 "nondisk",
2194 "inode",
2195 "rgrp",
2196 "meta",
2197 "iopen",
2198 "flock",
2199 "plock",
2200 "quota",
2201 "journal",
2202};
2203
2204static const char *gfs2_stype[] = {
2205 [GFS2_LKS_SRTT] = "srtt",
2206 [GFS2_LKS_SRTTVAR] = "srttvar",
2207 [GFS2_LKS_SRTTB] = "srttb",
2208 [GFS2_LKS_SRTTVARB] = "srttvarb",
2209 [GFS2_LKS_SIRT] = "sirt",
2210 [GFS2_LKS_SIRTVAR] = "sirtvar",
2211 [GFS2_LKS_DCOUNT] = "dlm",
2212 [GFS2_LKS_QCOUNT] = "queue",
2213};
2214
2215#define GFS2_NR_SBSTATS (ARRAY_SIZE(gfs2_gltype) * ARRAY_SIZE(gfs2_stype))
2216
2217static int gfs2_sbstats_seq_show(struct seq_file *seq, void *iter_ptr)
2218{
2219 struct gfs2_sbd *sdp = seq->private;
2220 loff_t pos = *(loff_t *)iter_ptr;
2221 unsigned index = pos >> 3;
2222 unsigned subindex = pos & 0x07;
2223 int i;
2224
2225 if (index == 0 && subindex != 0)
2226 return 0;
2227
2228 seq_printf(seq, "%-10s %8s:", gfs2_gltype[index],
2229 (index == 0) ? "cpu": gfs2_stype[subindex]);
2230
2231 for_each_possible_cpu(i) {
2232 const struct gfs2_pcpu_lkstats *lkstats = per_cpu_ptr(sdp->sd_lkstats, i);
2233
2234 if (index == 0)
2235 seq_printf(seq, " %15u", i);
2236 else
2237 seq_printf(seq, " %15llu", (unsigned long long)lkstats->
2238 lkstats[index - 1].stats[subindex]);
2239 }
2240 seq_putc(seq, '\n');
2241 return 0;
2242}
2243
2244int __init gfs2_glock_init(void)
2245{
2246 int i, ret;
2247
2248 ret = rhashtable_init(&gl_hash_table, &ht_parms);
2249 if (ret < 0)
2250 return ret;
2251
2252 glock_workqueue = alloc_workqueue("glock_workqueue", WQ_MEM_RECLAIM |
2253 WQ_HIGHPRI | WQ_FREEZABLE, 0);
2254 if (!glock_workqueue) {
2255 rhashtable_destroy(&gl_hash_table);
2256 return -ENOMEM;
2257 }
2258 gfs2_delete_workqueue = alloc_workqueue("delete_workqueue",
2259 WQ_MEM_RECLAIM | WQ_FREEZABLE,
2260 0);
2261 if (!gfs2_delete_workqueue) {
2262 destroy_workqueue(glock_workqueue);
2263 rhashtable_destroy(&gl_hash_table);
2264 return -ENOMEM;
2265 }
2266
2267 ret = register_shrinker(&glock_shrinker);
2268 if (ret) {
2269 destroy_workqueue(gfs2_delete_workqueue);
2270 destroy_workqueue(glock_workqueue);
2271 rhashtable_destroy(&gl_hash_table);
2272 return ret;
2273 }
2274
2275 for (i = 0; i < GLOCK_WAIT_TABLE_SIZE; i++)
2276 init_waitqueue_head(glock_wait_table + i);
2277
2278 return 0;
2279}
2280
2281void gfs2_glock_exit(void)
2282{
2283 unregister_shrinker(&glock_shrinker);
2284 rhashtable_destroy(&gl_hash_table);
2285 destroy_workqueue(glock_workqueue);
2286 destroy_workqueue(gfs2_delete_workqueue);
2287}
2288
2289static void gfs2_glock_iter_next(struct gfs2_glock_iter *gi, loff_t n)
2290{
2291 struct gfs2_glock *gl = gi->gl;
2292
2293 if (gl) {
2294 if (n == 0)
2295 return;
2296 if (!lockref_put_not_zero(&gl->gl_lockref))
2297 gfs2_glock_queue_put(gl);
2298 }
2299 for (;;) {
2300 gl = rhashtable_walk_next(&gi->hti);
2301 if (IS_ERR_OR_NULL(gl)) {
2302 if (gl == ERR_PTR(-EAGAIN)) {
2303 n = 1;
2304 continue;
2305 }
2306 gl = NULL;
2307 break;
2308 }
2309 if (gl->gl_name.ln_sbd != gi->sdp)
2310 continue;
2311 if (n <= 1) {
2312 if (!lockref_get_not_dead(&gl->gl_lockref))
2313 continue;
2314 break;
2315 } else {
2316 if (__lockref_is_dead(&gl->gl_lockref))
2317 continue;
2318 n--;
2319 }
2320 }
2321 gi->gl = gl;
2322}
2323
2324static void *gfs2_glock_seq_start(struct seq_file *seq, loff_t *pos)
2325 __acquires(RCU)
2326{
2327 struct gfs2_glock_iter *gi = seq->private;
2328 loff_t n;
2329
2330
2331
2332
2333
2334 if (*pos < gi->last_pos) {
2335 rhashtable_walk_exit(&gi->hti);
2336 rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2337 n = *pos + 1;
2338 } else {
2339 n = *pos - gi->last_pos;
2340 }
2341
2342 rhashtable_walk_start(&gi->hti);
2343
2344 gfs2_glock_iter_next(gi, n);
2345 gi->last_pos = *pos;
2346 return gi->gl;
2347}
2348
2349static void *gfs2_glock_seq_next(struct seq_file *seq, void *iter_ptr,
2350 loff_t *pos)
2351{
2352 struct gfs2_glock_iter *gi = seq->private;
2353
2354 (*pos)++;
2355 gi->last_pos = *pos;
2356 gfs2_glock_iter_next(gi, 1);
2357 return gi->gl;
2358}
2359
2360static void gfs2_glock_seq_stop(struct seq_file *seq, void *iter_ptr)
2361 __releases(RCU)
2362{
2363 struct gfs2_glock_iter *gi = seq->private;
2364
2365 rhashtable_walk_stop(&gi->hti);
2366}
2367
2368static int gfs2_glock_seq_show(struct seq_file *seq, void *iter_ptr)
2369{
2370 dump_glock(seq, iter_ptr, false);
2371 return 0;
2372}
2373
2374static void *gfs2_sbstats_seq_start(struct seq_file *seq, loff_t *pos)
2375{
2376 preempt_disable();
2377 if (*pos >= GFS2_NR_SBSTATS)
2378 return NULL;
2379 return pos;
2380}
2381
2382static void *gfs2_sbstats_seq_next(struct seq_file *seq, void *iter_ptr,
2383 loff_t *pos)
2384{
2385 (*pos)++;
2386 if (*pos >= GFS2_NR_SBSTATS)
2387 return NULL;
2388 return pos;
2389}
2390
2391static void gfs2_sbstats_seq_stop(struct seq_file *seq, void *iter_ptr)
2392{
2393 preempt_enable();
2394}
2395
2396static const struct seq_operations gfs2_glock_seq_ops = {
2397 .start = gfs2_glock_seq_start,
2398 .next = gfs2_glock_seq_next,
2399 .stop = gfs2_glock_seq_stop,
2400 .show = gfs2_glock_seq_show,
2401};
2402
2403static const struct seq_operations gfs2_glstats_seq_ops = {
2404 .start = gfs2_glock_seq_start,
2405 .next = gfs2_glock_seq_next,
2406 .stop = gfs2_glock_seq_stop,
2407 .show = gfs2_glstats_seq_show,
2408};
2409
2410static const struct seq_operations gfs2_sbstats_seq_ops = {
2411 .start = gfs2_sbstats_seq_start,
2412 .next = gfs2_sbstats_seq_next,
2413 .stop = gfs2_sbstats_seq_stop,
2414 .show = gfs2_sbstats_seq_show,
2415};
2416
2417#define GFS2_SEQ_GOODSIZE min(PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER, 65536UL)
2418
2419static int __gfs2_glocks_open(struct inode *inode, struct file *file,
2420 const struct seq_operations *ops)
2421{
2422 int ret = seq_open_private(file, ops, sizeof(struct gfs2_glock_iter));
2423 if (ret == 0) {
2424 struct seq_file *seq = file->private_data;
2425 struct gfs2_glock_iter *gi = seq->private;
2426
2427 gi->sdp = inode->i_private;
2428 seq->buf = kmalloc(GFS2_SEQ_GOODSIZE, GFP_KERNEL | __GFP_NOWARN);
2429 if (seq->buf)
2430 seq->size = GFS2_SEQ_GOODSIZE;
2431
2432
2433
2434
2435 gi->last_pos = -1;
2436 gi->gl = NULL;
2437 rhashtable_walk_enter(&gl_hash_table, &gi->hti);
2438 }
2439 return ret;
2440}
2441
2442static int gfs2_glocks_open(struct inode *inode, struct file *file)
2443{
2444 return __gfs2_glocks_open(inode, file, &gfs2_glock_seq_ops);
2445}
2446
2447static int gfs2_glocks_release(struct inode *inode, struct file *file)
2448{
2449 struct seq_file *seq = file->private_data;
2450 struct gfs2_glock_iter *gi = seq->private;
2451
2452 if (gi->gl)
2453 gfs2_glock_put(gi->gl);
2454 rhashtable_walk_exit(&gi->hti);
2455 return seq_release_private(inode, file);
2456}
2457
2458static int gfs2_glstats_open(struct inode *inode, struct file *file)
2459{
2460 return __gfs2_glocks_open(inode, file, &gfs2_glstats_seq_ops);
2461}
2462
2463static int gfs2_sbstats_open(struct inode *inode, struct file *file)
2464{
2465 int ret = seq_open(file, &gfs2_sbstats_seq_ops);
2466 if (ret == 0) {
2467 struct seq_file *seq = file->private_data;
2468 seq->private = inode->i_private;
2469 }
2470 return ret;
2471}
2472
2473static const struct file_operations gfs2_glocks_fops = {
2474 .owner = THIS_MODULE,
2475 .open = gfs2_glocks_open,
2476 .read = seq_read,
2477 .llseek = seq_lseek,
2478 .release = gfs2_glocks_release,
2479};
2480
2481static const struct file_operations gfs2_glstats_fops = {
2482 .owner = THIS_MODULE,
2483 .open = gfs2_glstats_open,
2484 .read = seq_read,
2485 .llseek = seq_lseek,
2486 .release = gfs2_glocks_release,
2487};
2488
2489static const struct file_operations gfs2_sbstats_fops = {
2490 .owner = THIS_MODULE,
2491 .open = gfs2_sbstats_open,
2492 .read = seq_read,
2493 .llseek = seq_lseek,
2494 .release = seq_release,
2495};
2496
2497void gfs2_create_debugfs_file(struct gfs2_sbd *sdp)
2498{
2499 sdp->debugfs_dir = debugfs_create_dir(sdp->sd_table_name, gfs2_root);
2500
2501 debugfs_create_file("glocks", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2502 &gfs2_glocks_fops);
2503
2504 debugfs_create_file("glstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2505 &gfs2_glstats_fops);
2506
2507 debugfs_create_file("sbstats", S_IFREG | S_IRUGO, sdp->debugfs_dir, sdp,
2508 &gfs2_sbstats_fops);
2509}
2510
2511void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp)
2512{
2513 debugfs_remove_recursive(sdp->debugfs_dir);
2514 sdp->debugfs_dir = NULL;
2515}
2516
2517void gfs2_register_debugfs(void)
2518{
2519 gfs2_root = debugfs_create_dir("gfs2", NULL);
2520}
2521
2522void gfs2_unregister_debugfs(void)
2523{
2524 debugfs_remove(gfs2_root);
2525 gfs2_root = NULL;
2526}
2527