1
2
3
4
5
6#include "xfs.h"
7#include "xfs_fs.h"
8#include "xfs_shared.h"
9#include "xfs_format.h"
10#include "xfs_log_format.h"
11#include "xfs_trans_resv.h"
12#include "xfs_mount.h"
13#include "xfs_inode.h"
14#include "xfs_trans.h"
15#include "xfs_trans_priv.h"
16#include "xfs_inode_item.h"
17#include "xfs_quota.h"
18#include "xfs_trace.h"
19#include "xfs_icache.h"
20#include "xfs_bmap_util.h"
21#include "xfs_dquot_item.h"
22#include "xfs_dquot.h"
23#include "xfs_reflink.h"
24#include "xfs_ialloc.h"
25#include "xfs_ag.h"
26
27#include <linux/iversion.h>
28
29
30
31
32#define XFS_ICI_RECLAIM_TAG 0
33
34#define XFS_ICI_BLOCKGC_TAG 1
35
36
37
38
39
40enum xfs_icwalk_goal {
41
42 XFS_ICWALK_BLOCKGC = XFS_ICI_BLOCKGC_TAG,
43 XFS_ICWALK_RECLAIM = XFS_ICI_RECLAIM_TAG,
44};
45
46static int xfs_icwalk(struct xfs_mount *mp,
47 enum xfs_icwalk_goal goal, struct xfs_icwalk *icw);
48static int xfs_icwalk_ag(struct xfs_perag *pag,
49 enum xfs_icwalk_goal goal, struct xfs_icwalk *icw);
50
51
52
53
54
55
56
57#define XFS_ICWALK_FLAG_SCAN_LIMIT (1U << 28)
58
59#define XFS_ICWALK_FLAG_RECLAIM_SICK (1U << 27)
60#define XFS_ICWALK_FLAG_UNION (1U << 26)
61
62#define XFS_ICWALK_PRIVATE_FLAGS (XFS_ICWALK_FLAG_SCAN_LIMIT | \
63 XFS_ICWALK_FLAG_RECLAIM_SICK | \
64 XFS_ICWALK_FLAG_UNION)
65
66
67
68
69struct xfs_inode *
70xfs_inode_alloc(
71 struct xfs_mount *mp,
72 xfs_ino_t ino)
73{
74 struct xfs_inode *ip;
75
76
77
78
79
80 ip = kmem_cache_alloc(xfs_inode_zone, GFP_KERNEL | __GFP_NOFAIL);
81
82 if (inode_init_always(mp->m_super, VFS_I(ip))) {
83 kmem_cache_free(xfs_inode_zone, ip);
84 return NULL;
85 }
86
87
88 VFS_I(ip)->i_mode = 0;
89 VFS_I(ip)->i_state = 0;
90
91 XFS_STATS_INC(mp, vn_active);
92 ASSERT(atomic_read(&ip->i_pincount) == 0);
93 ASSERT(ip->i_ino == 0);
94
95
96 ip->i_ino = ino;
97 ip->i_mount = mp;
98 memset(&ip->i_imap, 0, sizeof(struct xfs_imap));
99 ip->i_afp = NULL;
100 ip->i_cowfp = NULL;
101 memset(&ip->i_df, 0, sizeof(ip->i_df));
102 ip->i_flags = 0;
103 ip->i_delayed_blks = 0;
104 ip->i_diflags2 = mp->m_ino_geo.new_diflags2;
105 ip->i_nblocks = 0;
106 ip->i_forkoff = 0;
107 ip->i_sick = 0;
108 ip->i_checked = 0;
109 INIT_WORK(&ip->i_ioend_work, xfs_end_io);
110 INIT_LIST_HEAD(&ip->i_ioend_list);
111 spin_lock_init(&ip->i_ioend_lock);
112
113 return ip;
114}
115
116STATIC void
117xfs_inode_free_callback(
118 struct rcu_head *head)
119{
120 struct inode *inode = container_of(head, struct inode, i_rcu);
121 struct xfs_inode *ip = XFS_I(inode);
122
123 switch (VFS_I(ip)->i_mode & S_IFMT) {
124 case S_IFREG:
125 case S_IFDIR:
126 case S_IFLNK:
127 xfs_idestroy_fork(&ip->i_df);
128 break;
129 }
130
131 if (ip->i_afp) {
132 xfs_idestroy_fork(ip->i_afp);
133 kmem_cache_free(xfs_ifork_zone, ip->i_afp);
134 }
135 if (ip->i_cowfp) {
136 xfs_idestroy_fork(ip->i_cowfp);
137 kmem_cache_free(xfs_ifork_zone, ip->i_cowfp);
138 }
139 if (ip->i_itemp) {
140 ASSERT(!test_bit(XFS_LI_IN_AIL,
141 &ip->i_itemp->ili_item.li_flags));
142 xfs_inode_item_destroy(ip);
143 ip->i_itemp = NULL;
144 }
145
146 kmem_cache_free(xfs_inode_zone, ip);
147}
148
149static void
150__xfs_inode_free(
151 struct xfs_inode *ip)
152{
153
154 ASSERT(atomic_read(&ip->i_pincount) == 0);
155 ASSERT(!ip->i_itemp || list_empty(&ip->i_itemp->ili_item.li_bio_list));
156 XFS_STATS_DEC(ip->i_mount, vn_active);
157
158 call_rcu(&VFS_I(ip)->i_rcu, xfs_inode_free_callback);
159}
160
161void
162xfs_inode_free(
163 struct xfs_inode *ip)
164{
165 ASSERT(!xfs_iflags_test(ip, XFS_IFLUSHING));
166
167
168
169
170
171
172
173 spin_lock(&ip->i_flags_lock);
174 ip->i_flags = XFS_IRECLAIM;
175 ip->i_ino = 0;
176 spin_unlock(&ip->i_flags_lock);
177
178 __xfs_inode_free(ip);
179}
180
181
182
183
184
185static void
186xfs_reclaim_work_queue(
187 struct xfs_mount *mp)
188{
189
190 rcu_read_lock();
191 if (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_RECLAIM_TAG)) {
192 queue_delayed_work(mp->m_reclaim_workqueue, &mp->m_reclaim_work,
193 msecs_to_jiffies(xfs_syncd_centisecs / 6 * 10));
194 }
195 rcu_read_unlock();
196}
197
198
199
200
201
202static inline void
203xfs_blockgc_queue(
204 struct xfs_perag *pag)
205{
206 struct xfs_mount *mp = pag->pag_mount;
207
208 if (!xfs_is_blockgc_enabled(mp))
209 return;
210
211 rcu_read_lock();
212 if (radix_tree_tagged(&pag->pag_ici_root, XFS_ICI_BLOCKGC_TAG))
213 queue_delayed_work(pag->pag_mount->m_blockgc_wq,
214 &pag->pag_blockgc_work,
215 msecs_to_jiffies(xfs_blockgc_secs * 1000));
216 rcu_read_unlock();
217}
218
219
220static void
221xfs_perag_set_inode_tag(
222 struct xfs_perag *pag,
223 xfs_agino_t agino,
224 unsigned int tag)
225{
226 struct xfs_mount *mp = pag->pag_mount;
227 bool was_tagged;
228
229 lockdep_assert_held(&pag->pag_ici_lock);
230
231 was_tagged = radix_tree_tagged(&pag->pag_ici_root, tag);
232 radix_tree_tag_set(&pag->pag_ici_root, agino, tag);
233
234 if (tag == XFS_ICI_RECLAIM_TAG)
235 pag->pag_ici_reclaimable++;
236
237 if (was_tagged)
238 return;
239
240
241 spin_lock(&mp->m_perag_lock);
242 radix_tree_tag_set(&mp->m_perag_tree, pag->pag_agno, tag);
243 spin_unlock(&mp->m_perag_lock);
244
245
246 switch (tag) {
247 case XFS_ICI_RECLAIM_TAG:
248 xfs_reclaim_work_queue(mp);
249 break;
250 case XFS_ICI_BLOCKGC_TAG:
251 xfs_blockgc_queue(pag);
252 break;
253 }
254
255 trace_xfs_perag_set_inode_tag(mp, pag->pag_agno, tag, _RET_IP_);
256}
257
258
259static void
260xfs_perag_clear_inode_tag(
261 struct xfs_perag *pag,
262 xfs_agino_t agino,
263 unsigned int tag)
264{
265 struct xfs_mount *mp = pag->pag_mount;
266
267 lockdep_assert_held(&pag->pag_ici_lock);
268
269
270
271
272
273 if (agino != NULLAGINO)
274 radix_tree_tag_clear(&pag->pag_ici_root, agino, tag);
275 else
276 ASSERT(tag == XFS_ICI_RECLAIM_TAG);
277
278 if (tag == XFS_ICI_RECLAIM_TAG)
279 pag->pag_ici_reclaimable--;
280
281 if (radix_tree_tagged(&pag->pag_ici_root, tag))
282 return;
283
284
285 spin_lock(&mp->m_perag_lock);
286 radix_tree_tag_clear(&mp->m_perag_tree, pag->pag_agno, tag);
287 spin_unlock(&mp->m_perag_lock);
288
289 trace_xfs_perag_clear_inode_tag(mp, pag->pag_agno, tag, _RET_IP_);
290}
291
292static inline void
293xfs_inew_wait(
294 struct xfs_inode *ip)
295{
296 wait_queue_head_t *wq = bit_waitqueue(&ip->i_flags, __XFS_INEW_BIT);
297 DEFINE_WAIT_BIT(wait, &ip->i_flags, __XFS_INEW_BIT);
298
299 do {
300 prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
301 if (!xfs_iflags_test(ip, XFS_INEW))
302 break;
303 schedule();
304 } while (true);
305 finish_wait(wq, &wait.wq_entry);
306}
307
308
309
310
311
312
313
314
315
316static int
317xfs_reinit_inode(
318 struct xfs_mount *mp,
319 struct inode *inode)
320{
321 int error;
322 uint32_t nlink = inode->i_nlink;
323 uint32_t generation = inode->i_generation;
324 uint64_t version = inode_peek_iversion(inode);
325 umode_t mode = inode->i_mode;
326 dev_t dev = inode->i_rdev;
327 kuid_t uid = inode->i_uid;
328 kgid_t gid = inode->i_gid;
329
330 error = inode_init_always(mp->m_super, inode);
331
332 set_nlink(inode, nlink);
333 inode->i_generation = generation;
334 inode_set_iversion_queried(inode, version);
335 inode->i_mode = mode;
336 inode->i_rdev = dev;
337 inode->i_uid = uid;
338 inode->i_gid = gid;
339 return error;
340}
341
342
343
344
345
346static int
347xfs_iget_recycle(
348 struct xfs_perag *pag,
349 struct xfs_inode *ip) __releases(&ip->i_flags_lock)
350{
351 struct xfs_mount *mp = ip->i_mount;
352 struct inode *inode = VFS_I(ip);
353 int error;
354
355 trace_xfs_iget_recycle(ip);
356
357
358
359
360
361
362
363 ip->i_flags |= XFS_IRECLAIM;
364
365 spin_unlock(&ip->i_flags_lock);
366 rcu_read_unlock();
367
368 ASSERT(!rwsem_is_locked(&inode->i_rwsem));
369 error = xfs_reinit_inode(mp, inode);
370 if (error) {
371 bool wake;
372
373
374
375
376
377 rcu_read_lock();
378 spin_lock(&ip->i_flags_lock);
379 wake = !!__xfs_iflags_test(ip, XFS_INEW);
380 ip->i_flags &= ~(XFS_INEW | XFS_IRECLAIM);
381 if (wake)
382 wake_up_bit(&ip->i_flags, __XFS_INEW_BIT);
383 ASSERT(ip->i_flags & XFS_IRECLAIMABLE);
384 spin_unlock(&ip->i_flags_lock);
385 rcu_read_unlock();
386
387 trace_xfs_iget_recycle_fail(ip);
388 return error;
389 }
390
391 spin_lock(&pag->pag_ici_lock);
392 spin_lock(&ip->i_flags_lock);
393
394
395
396
397
398
399 ip->i_flags &= ~XFS_IRECLAIM_RESET_FLAGS;
400 ip->i_flags |= XFS_INEW;
401 xfs_perag_clear_inode_tag(pag, XFS_INO_TO_AGINO(mp, ip->i_ino),
402 XFS_ICI_RECLAIM_TAG);
403 inode->i_state = I_NEW;
404 spin_unlock(&ip->i_flags_lock);
405 spin_unlock(&pag->pag_ici_lock);
406
407 return 0;
408}
409
410
411
412
413
414
415
416
417
418
419
420static int
421xfs_iget_check_free_state(
422 struct xfs_inode *ip,
423 int flags)
424{
425 if (flags & XFS_IGET_CREATE) {
426
427 if (VFS_I(ip)->i_mode != 0) {
428 xfs_warn(ip->i_mount,
429"Corruption detected! Free inode 0x%llx not marked free! (mode 0x%x)",
430 ip->i_ino, VFS_I(ip)->i_mode);
431 return -EFSCORRUPTED;
432 }
433
434 if (ip->i_nblocks != 0) {
435 xfs_warn(ip->i_mount,
436"Corruption detected! Free inode 0x%llx has blocks allocated!",
437 ip->i_ino);
438 return -EFSCORRUPTED;
439 }
440 return 0;
441 }
442
443
444 if (VFS_I(ip)->i_mode == 0)
445 return -ENOENT;
446
447 return 0;
448}
449
450
451static void
452xfs_inodegc_queue_all(
453 struct xfs_mount *mp)
454{
455 struct xfs_inodegc *gc;
456 int cpu;
457
458 for_each_online_cpu(cpu) {
459 gc = per_cpu_ptr(mp->m_inodegc, cpu);
460 if (!llist_empty(&gc->list))
461 queue_work_on(cpu, mp->m_inodegc_wq, &gc->work);
462 }
463}
464
465
466
467
468static int
469xfs_iget_cache_hit(
470 struct xfs_perag *pag,
471 struct xfs_inode *ip,
472 xfs_ino_t ino,
473 int flags,
474 int lock_flags) __releases(RCU)
475{
476 struct inode *inode = VFS_I(ip);
477 struct xfs_mount *mp = ip->i_mount;
478 int error;
479
480
481
482
483
484
485
486
487 spin_lock(&ip->i_flags_lock);
488 if (ip->i_ino != ino)
489 goto out_skip;
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509 if (ip->i_flags & (XFS_INEW | XFS_IRECLAIM | XFS_INACTIVATING))
510 goto out_skip;
511
512 if (ip->i_flags & XFS_NEED_INACTIVE) {
513
514 if (VFS_I(ip)->i_nlink == 0) {
515 error = -ENOENT;
516 goto out_error;
517 }
518 goto out_inodegc_flush;
519 }
520
521
522
523
524
525 error = xfs_iget_check_free_state(ip, flags);
526 if (error)
527 goto out_error;
528
529
530 if ((flags & XFS_IGET_INCORE) &&
531 (ip->i_flags & XFS_IRECLAIMABLE))
532 goto out_skip;
533
534
535 if (ip->i_flags & XFS_IRECLAIMABLE) {
536
537 error = xfs_iget_recycle(pag, ip);
538 if (error)
539 return error;
540 } else {
541
542 if (!igrab(inode))
543 goto out_skip;
544
545
546 spin_unlock(&ip->i_flags_lock);
547 rcu_read_unlock();
548 trace_xfs_iget_hit(ip);
549 }
550
551 if (lock_flags != 0)
552 xfs_ilock(ip, lock_flags);
553
554 if (!(flags & XFS_IGET_INCORE))
555 xfs_iflags_clear(ip, XFS_ISTALE);
556 XFS_STATS_INC(mp, xs_ig_found);
557
558 return 0;
559
560out_skip:
561 trace_xfs_iget_skip(ip);
562 XFS_STATS_INC(mp, xs_ig_frecycle);
563 error = -EAGAIN;
564out_error:
565 spin_unlock(&ip->i_flags_lock);
566 rcu_read_unlock();
567 return error;
568
569out_inodegc_flush:
570 spin_unlock(&ip->i_flags_lock);
571 rcu_read_unlock();
572
573
574
575
576 if (xfs_is_inodegc_enabled(mp))
577 xfs_inodegc_queue_all(mp);
578 return -EAGAIN;
579}
580
581static int
582xfs_iget_cache_miss(
583 struct xfs_mount *mp,
584 struct xfs_perag *pag,
585 xfs_trans_t *tp,
586 xfs_ino_t ino,
587 struct xfs_inode **ipp,
588 int flags,
589 int lock_flags)
590{
591 struct xfs_inode *ip;
592 int error;
593 xfs_agino_t agino = XFS_INO_TO_AGINO(mp, ino);
594 int iflags;
595
596 ip = xfs_inode_alloc(mp, ino);
597 if (!ip)
598 return -ENOMEM;
599
600 error = xfs_imap(mp, tp, ip->i_ino, &ip->i_imap, flags);
601 if (error)
602 goto out_destroy;
603
604
605
606
607
608
609
610
611
612
613
614 if (xfs_has_v3inodes(mp) &&
615 (flags & XFS_IGET_CREATE) && !xfs_has_ikeep(mp)) {
616 VFS_I(ip)->i_generation = prandom_u32();
617 } else {
618 struct xfs_buf *bp;
619
620 error = xfs_imap_to_bp(mp, tp, &ip->i_imap, &bp);
621 if (error)
622 goto out_destroy;
623
624 error = xfs_inode_from_disk(ip,
625 xfs_buf_offset(bp, ip->i_imap.im_boffset));
626 if (!error)
627 xfs_buf_set_ref(bp, XFS_INO_REF);
628 xfs_trans_brelse(tp, bp);
629
630 if (error)
631 goto out_destroy;
632 }
633
634 trace_xfs_iget_miss(ip);
635
636
637
638
639
640 error = xfs_iget_check_free_state(ip, flags);
641 if (error)
642 goto out_destroy;
643
644
645
646
647
648
649
650 if (radix_tree_preload(GFP_NOFS)) {
651 error = -EAGAIN;
652 goto out_destroy;
653 }
654
655
656
657
658
659 if (lock_flags) {
660 if (!xfs_ilock_nowait(ip, lock_flags))
661 BUG();
662 }
663
664
665
666
667
668
669
670
671
672
673 iflags = XFS_INEW;
674 if (flags & XFS_IGET_DONTCACHE)
675 d_mark_dontcache(VFS_I(ip));
676 ip->i_udquot = NULL;
677 ip->i_gdquot = NULL;
678 ip->i_pdquot = NULL;
679 xfs_iflags_set(ip, iflags);
680
681
682 spin_lock(&pag->pag_ici_lock);
683 error = radix_tree_insert(&pag->pag_ici_root, agino, ip);
684 if (unlikely(error)) {
685 WARN_ON(error != -EEXIST);
686 XFS_STATS_INC(mp, xs_ig_dup);
687 error = -EAGAIN;
688 goto out_preload_end;
689 }
690 spin_unlock(&pag->pag_ici_lock);
691 radix_tree_preload_end();
692
693 *ipp = ip;
694 return 0;
695
696out_preload_end:
697 spin_unlock(&pag->pag_ici_lock);
698 radix_tree_preload_end();
699 if (lock_flags)
700 xfs_iunlock(ip, lock_flags);
701out_destroy:
702 __destroy_inode(VFS_I(ip));
703 xfs_inode_free(ip);
704 return error;
705}
706
707
708
709
710
711
712
713
714
715
716
717
718
719int
720xfs_iget(
721 struct xfs_mount *mp,
722 struct xfs_trans *tp,
723 xfs_ino_t ino,
724 uint flags,
725 uint lock_flags,
726 struct xfs_inode **ipp)
727{
728 struct xfs_inode *ip;
729 struct xfs_perag *pag;
730 xfs_agino_t agino;
731 int error;
732
733 ASSERT((lock_flags & (XFS_IOLOCK_EXCL | XFS_IOLOCK_SHARED)) == 0);
734
735
736 if (!ino || XFS_INO_TO_AGNO(mp, ino) >= mp->m_sb.sb_agcount)
737 return -EINVAL;
738
739 XFS_STATS_INC(mp, xs_ig_attempts);
740
741
742 pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ino));
743 agino = XFS_INO_TO_AGINO(mp, ino);
744
745again:
746 error = 0;
747 rcu_read_lock();
748 ip = radix_tree_lookup(&pag->pag_ici_root, agino);
749
750 if (ip) {
751 error = xfs_iget_cache_hit(pag, ip, ino, flags, lock_flags);
752 if (error)
753 goto out_error_or_again;
754 } else {
755 rcu_read_unlock();
756 if (flags & XFS_IGET_INCORE) {
757 error = -ENODATA;
758 goto out_error_or_again;
759 }
760 XFS_STATS_INC(mp, xs_ig_missed);
761
762 error = xfs_iget_cache_miss(mp, pag, tp, ino, &ip,
763 flags, lock_flags);
764 if (error)
765 goto out_error_or_again;
766 }
767 xfs_perag_put(pag);
768
769 *ipp = ip;
770
771
772
773
774
775 if (xfs_iflags_test(ip, XFS_INEW) && VFS_I(ip)->i_mode != 0)
776 xfs_setup_existing_inode(ip);
777 return 0;
778
779out_error_or_again:
780 if (!(flags & XFS_IGET_INCORE) && error == -EAGAIN) {
781 delay(1);
782 goto again;
783 }
784 xfs_perag_put(pag);
785 return error;
786}
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807int
808xfs_icache_inode_is_allocated(
809 struct xfs_mount *mp,
810 struct xfs_trans *tp,
811 xfs_ino_t ino,
812 bool *inuse)
813{
814 struct xfs_inode *ip;
815 int error;
816
817 error = xfs_iget(mp, tp, ino, XFS_IGET_INCORE, 0, &ip);
818 if (error)
819 return error;
820
821 *inuse = !!(VFS_I(ip)->i_mode);
822 xfs_irele(ip);
823 return 0;
824}
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843static bool
844xfs_reclaim_igrab(
845 struct xfs_inode *ip,
846 struct xfs_icwalk *icw)
847{
848 ASSERT(rcu_read_lock_held());
849
850 spin_lock(&ip->i_flags_lock);
851 if (!__xfs_iflags_test(ip, XFS_IRECLAIMABLE) ||
852 __xfs_iflags_test(ip, XFS_IRECLAIM)) {
853
854 spin_unlock(&ip->i_flags_lock);
855 return false;
856 }
857
858
859 if (ip->i_sick &&
860 (!icw || !(icw->icw_flags & XFS_ICWALK_FLAG_RECLAIM_SICK))) {
861 spin_unlock(&ip->i_flags_lock);
862 return false;
863 }
864
865 __xfs_iflags_set(ip, XFS_IRECLAIM);
866 spin_unlock(&ip->i_flags_lock);
867 return true;
868}
869
870
871
872
873
874
875
876
877
878
879
880
881
882static void
883xfs_reclaim_inode(
884 struct xfs_inode *ip,
885 struct xfs_perag *pag)
886{
887 xfs_ino_t ino = ip->i_ino;
888
889 if (!xfs_ilock_nowait(ip, XFS_ILOCK_EXCL))
890 goto out;
891 if (xfs_iflags_test_and_set(ip, XFS_IFLUSHING))
892 goto out_iunlock;
893
894 if (xfs_is_shutdown(ip->i_mount)) {
895 xfs_iunpin_wait(ip);
896 xfs_iflush_abort(ip);
897 goto reclaim;
898 }
899 if (xfs_ipincount(ip))
900 goto out_clear_flush;
901 if (!xfs_inode_clean(ip))
902 goto out_clear_flush;
903
904 xfs_iflags_clear(ip, XFS_IFLUSHING);
905reclaim:
906 trace_xfs_inode_reclaiming(ip);
907
908
909
910
911
912
913
914
915
916
917
918 spin_lock(&ip->i_flags_lock);
919 ip->i_flags = XFS_IRECLAIM;
920 ip->i_ino = 0;
921 ip->i_sick = 0;
922 ip->i_checked = 0;
923 spin_unlock(&ip->i_flags_lock);
924
925 xfs_iunlock(ip, XFS_ILOCK_EXCL);
926
927 XFS_STATS_INC(ip->i_mount, xs_ig_reclaims);
928
929
930
931
932
933
934
935 spin_lock(&pag->pag_ici_lock);
936 if (!radix_tree_delete(&pag->pag_ici_root,
937 XFS_INO_TO_AGINO(ip->i_mount, ino)))
938 ASSERT(0);
939 xfs_perag_clear_inode_tag(pag, NULLAGINO, XFS_ICI_RECLAIM_TAG);
940 spin_unlock(&pag->pag_ici_lock);
941
942
943
944
945
946
947
948
949
950 xfs_ilock(ip, XFS_ILOCK_EXCL);
951 ASSERT(!ip->i_udquot && !ip->i_gdquot && !ip->i_pdquot);
952 xfs_iunlock(ip, XFS_ILOCK_EXCL);
953 ASSERT(xfs_inode_clean(ip));
954
955 __xfs_inode_free(ip);
956 return;
957
958out_clear_flush:
959 xfs_iflags_clear(ip, XFS_IFLUSHING);
960out_iunlock:
961 xfs_iunlock(ip, XFS_ILOCK_EXCL);
962out:
963 xfs_iflags_clear(ip, XFS_IRECLAIM);
964}
965
966
967static inline bool
968xfs_want_reclaim_sick(
969 struct xfs_mount *mp)
970{
971 return xfs_is_unmounting(mp) || xfs_has_norecovery(mp) ||
972 xfs_is_shutdown(mp);
973}
974
975void
976xfs_reclaim_inodes(
977 struct xfs_mount *mp)
978{
979 struct xfs_icwalk icw = {
980 .icw_flags = 0,
981 };
982
983 if (xfs_want_reclaim_sick(mp))
984 icw.icw_flags |= XFS_ICWALK_FLAG_RECLAIM_SICK;
985
986 while (radix_tree_tagged(&mp->m_perag_tree, XFS_ICI_RECLAIM_TAG)) {
987 xfs_ail_push_all_sync(mp->m_ail);
988 xfs_icwalk(mp, XFS_ICWALK_RECLAIM, &icw);
989 }
990}
991
992
993
994
995
996
997
998
999long
1000xfs_reclaim_inodes_nr(
1001 struct xfs_mount *mp,
1002 unsigned long nr_to_scan)
1003{
1004 struct xfs_icwalk icw = {
1005 .icw_flags = XFS_ICWALK_FLAG_SCAN_LIMIT,
1006 .icw_scan_limit = min_t(unsigned long, LONG_MAX, nr_to_scan),
1007 };
1008
1009 if (xfs_want_reclaim_sick(mp))
1010 icw.icw_flags |= XFS_ICWALK_FLAG_RECLAIM_SICK;
1011
1012
1013 xfs_reclaim_work_queue(mp);
1014 xfs_ail_push_all(mp->m_ail);
1015
1016 xfs_icwalk(mp, XFS_ICWALK_RECLAIM, &icw);
1017 return 0;
1018}
1019
1020
1021
1022
1023
1024long
1025xfs_reclaim_inodes_count(
1026 struct xfs_mount *mp)
1027{
1028 struct xfs_perag *pag;
1029 xfs_agnumber_t ag = 0;
1030 long reclaimable = 0;
1031
1032 while ((pag = xfs_perag_get_tag(mp, ag, XFS_ICI_RECLAIM_TAG))) {
1033 ag = pag->pag_agno + 1;
1034 reclaimable += pag->pag_ici_reclaimable;
1035 xfs_perag_put(pag);
1036 }
1037 return reclaimable;
1038}
1039
1040STATIC bool
1041xfs_icwalk_match_id(
1042 struct xfs_inode *ip,
1043 struct xfs_icwalk *icw)
1044{
1045 if ((icw->icw_flags & XFS_ICWALK_FLAG_UID) &&
1046 !uid_eq(VFS_I(ip)->i_uid, icw->icw_uid))
1047 return false;
1048
1049 if ((icw->icw_flags & XFS_ICWALK_FLAG_GID) &&
1050 !gid_eq(VFS_I(ip)->i_gid, icw->icw_gid))
1051 return false;
1052
1053 if ((icw->icw_flags & XFS_ICWALK_FLAG_PRID) &&
1054 ip->i_projid != icw->icw_prid)
1055 return false;
1056
1057 return true;
1058}
1059
1060
1061
1062
1063
1064STATIC bool
1065xfs_icwalk_match_id_union(
1066 struct xfs_inode *ip,
1067 struct xfs_icwalk *icw)
1068{
1069 if ((icw->icw_flags & XFS_ICWALK_FLAG_UID) &&
1070 uid_eq(VFS_I(ip)->i_uid, icw->icw_uid))
1071 return true;
1072
1073 if ((icw->icw_flags & XFS_ICWALK_FLAG_GID) &&
1074 gid_eq(VFS_I(ip)->i_gid, icw->icw_gid))
1075 return true;
1076
1077 if ((icw->icw_flags & XFS_ICWALK_FLAG_PRID) &&
1078 ip->i_projid == icw->icw_prid)
1079 return true;
1080
1081 return false;
1082}
1083
1084
1085
1086
1087
1088
1089static bool
1090xfs_icwalk_match(
1091 struct xfs_inode *ip,
1092 struct xfs_icwalk *icw)
1093{
1094 bool match;
1095
1096 if (!icw)
1097 return true;
1098
1099 if (icw->icw_flags & XFS_ICWALK_FLAG_UNION)
1100 match = xfs_icwalk_match_id_union(ip, icw);
1101 else
1102 match = xfs_icwalk_match_id(ip, icw);
1103 if (!match)
1104 return false;
1105
1106
1107 if ((icw->icw_flags & XFS_ICWALK_FLAG_MINFILESIZE) &&
1108 XFS_ISIZE(ip) < icw->icw_min_file_size)
1109 return false;
1110
1111 return true;
1112}
1113
1114
1115
1116
1117
1118
1119
1120void
1121xfs_reclaim_worker(
1122 struct work_struct *work)
1123{
1124 struct xfs_mount *mp = container_of(to_delayed_work(work),
1125 struct xfs_mount, m_reclaim_work);
1126
1127 xfs_icwalk(mp, XFS_ICWALK_RECLAIM, NULL);
1128 xfs_reclaim_work_queue(mp);
1129}
1130
1131STATIC int
1132xfs_inode_free_eofblocks(
1133 struct xfs_inode *ip,
1134 struct xfs_icwalk *icw,
1135 unsigned int *lockflags)
1136{
1137 bool wait;
1138
1139 wait = icw && (icw->icw_flags & XFS_ICWALK_FLAG_SYNC);
1140
1141 if (!xfs_iflags_test(ip, XFS_IEOFBLOCKS))
1142 return 0;
1143
1144
1145
1146
1147
1148 if (!wait && mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY))
1149 return 0;
1150
1151 if (!xfs_icwalk_match(ip, icw))
1152 return 0;
1153
1154
1155
1156
1157
1158 if (!xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) {
1159 if (wait)
1160 return -EAGAIN;
1161 return 0;
1162 }
1163 *lockflags |= XFS_IOLOCK_EXCL;
1164
1165 if (xfs_can_free_eofblocks(ip, false))
1166 return xfs_free_eofblocks(ip);
1167
1168
1169 trace_xfs_inode_free_eofblocks_invalid(ip);
1170 xfs_inode_clear_eofblocks_tag(ip);
1171 return 0;
1172}
1173
1174static void
1175xfs_blockgc_set_iflag(
1176 struct xfs_inode *ip,
1177 unsigned long iflag)
1178{
1179 struct xfs_mount *mp = ip->i_mount;
1180 struct xfs_perag *pag;
1181
1182 ASSERT((iflag & ~(XFS_IEOFBLOCKS | XFS_ICOWBLOCKS)) == 0);
1183
1184
1185
1186
1187
1188 if (ip->i_flags & iflag)
1189 return;
1190 spin_lock(&ip->i_flags_lock);
1191 ip->i_flags |= iflag;
1192 spin_unlock(&ip->i_flags_lock);
1193
1194 pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
1195 spin_lock(&pag->pag_ici_lock);
1196
1197 xfs_perag_set_inode_tag(pag, XFS_INO_TO_AGINO(mp, ip->i_ino),
1198 XFS_ICI_BLOCKGC_TAG);
1199
1200 spin_unlock(&pag->pag_ici_lock);
1201 xfs_perag_put(pag);
1202}
1203
1204void
1205xfs_inode_set_eofblocks_tag(
1206 xfs_inode_t *ip)
1207{
1208 trace_xfs_inode_set_eofblocks_tag(ip);
1209 return xfs_blockgc_set_iflag(ip, XFS_IEOFBLOCKS);
1210}
1211
1212static void
1213xfs_blockgc_clear_iflag(
1214 struct xfs_inode *ip,
1215 unsigned long iflag)
1216{
1217 struct xfs_mount *mp = ip->i_mount;
1218 struct xfs_perag *pag;
1219 bool clear_tag;
1220
1221 ASSERT((iflag & ~(XFS_IEOFBLOCKS | XFS_ICOWBLOCKS)) == 0);
1222
1223 spin_lock(&ip->i_flags_lock);
1224 ip->i_flags &= ~iflag;
1225 clear_tag = (ip->i_flags & (XFS_IEOFBLOCKS | XFS_ICOWBLOCKS)) == 0;
1226 spin_unlock(&ip->i_flags_lock);
1227
1228 if (!clear_tag)
1229 return;
1230
1231 pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
1232 spin_lock(&pag->pag_ici_lock);
1233
1234 xfs_perag_clear_inode_tag(pag, XFS_INO_TO_AGINO(mp, ip->i_ino),
1235 XFS_ICI_BLOCKGC_TAG);
1236
1237 spin_unlock(&pag->pag_ici_lock);
1238 xfs_perag_put(pag);
1239}
1240
1241void
1242xfs_inode_clear_eofblocks_tag(
1243 xfs_inode_t *ip)
1244{
1245 trace_xfs_inode_clear_eofblocks_tag(ip);
1246 return xfs_blockgc_clear_iflag(ip, XFS_IEOFBLOCKS);
1247}
1248
1249
1250
1251
1252
1253
1254static bool
1255xfs_prep_free_cowblocks(
1256 struct xfs_inode *ip)
1257{
1258
1259
1260
1261
1262 if (!xfs_inode_has_cow_data(ip)) {
1263 trace_xfs_inode_free_cowblocks_invalid(ip);
1264 xfs_inode_clear_cowblocks_tag(ip);
1265 return false;
1266 }
1267
1268
1269
1270
1271
1272 if ((VFS_I(ip)->i_state & I_DIRTY_PAGES) ||
1273 mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_DIRTY) ||
1274 mapping_tagged(VFS_I(ip)->i_mapping, PAGECACHE_TAG_WRITEBACK) ||
1275 atomic_read(&VFS_I(ip)->i_dio_count))
1276 return false;
1277
1278 return true;
1279}
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293STATIC int
1294xfs_inode_free_cowblocks(
1295 struct xfs_inode *ip,
1296 struct xfs_icwalk *icw,
1297 unsigned int *lockflags)
1298{
1299 bool wait;
1300 int ret = 0;
1301
1302 wait = icw && (icw->icw_flags & XFS_ICWALK_FLAG_SYNC);
1303
1304 if (!xfs_iflags_test(ip, XFS_ICOWBLOCKS))
1305 return 0;
1306
1307 if (!xfs_prep_free_cowblocks(ip))
1308 return 0;
1309
1310 if (!xfs_icwalk_match(ip, icw))
1311 return 0;
1312
1313
1314
1315
1316
1317 if (!(*lockflags & XFS_IOLOCK_EXCL) &&
1318 !xfs_ilock_nowait(ip, XFS_IOLOCK_EXCL)) {
1319 if (wait)
1320 return -EAGAIN;
1321 return 0;
1322 }
1323 *lockflags |= XFS_IOLOCK_EXCL;
1324
1325 if (!xfs_ilock_nowait(ip, XFS_MMAPLOCK_EXCL)) {
1326 if (wait)
1327 return -EAGAIN;
1328 return 0;
1329 }
1330 *lockflags |= XFS_MMAPLOCK_EXCL;
1331
1332
1333
1334
1335
1336 if (xfs_prep_free_cowblocks(ip))
1337 ret = xfs_reflink_cancel_cow_range(ip, 0, NULLFILEOFF, false);
1338 return ret;
1339}
1340
1341void
1342xfs_inode_set_cowblocks_tag(
1343 xfs_inode_t *ip)
1344{
1345 trace_xfs_inode_set_cowblocks_tag(ip);
1346 return xfs_blockgc_set_iflag(ip, XFS_ICOWBLOCKS);
1347}
1348
1349void
1350xfs_inode_clear_cowblocks_tag(
1351 xfs_inode_t *ip)
1352{
1353 trace_xfs_inode_clear_cowblocks_tag(ip);
1354 return xfs_blockgc_clear_iflag(ip, XFS_ICOWBLOCKS);
1355}
1356
1357
1358void
1359xfs_blockgc_stop(
1360 struct xfs_mount *mp)
1361{
1362 struct xfs_perag *pag;
1363 xfs_agnumber_t agno;
1364
1365 if (!xfs_clear_blockgc_enabled(mp))
1366 return;
1367
1368 for_each_perag(mp, agno, pag)
1369 cancel_delayed_work_sync(&pag->pag_blockgc_work);
1370 trace_xfs_blockgc_stop(mp, __return_address);
1371}
1372
1373
1374void
1375xfs_blockgc_start(
1376 struct xfs_mount *mp)
1377{
1378 struct xfs_perag *pag;
1379 xfs_agnumber_t agno;
1380
1381 if (xfs_set_blockgc_enabled(mp))
1382 return;
1383
1384 trace_xfs_blockgc_start(mp, __return_address);
1385 for_each_perag_tag(mp, agno, pag, XFS_ICI_BLOCKGC_TAG)
1386 xfs_blockgc_queue(pag);
1387}
1388
1389
1390#define XFS_BLOCKGC_NOGRAB_IFLAGS (XFS_INEW | \
1391 XFS_NEED_INACTIVE | \
1392 XFS_INACTIVATING | \
1393 XFS_IRECLAIMABLE | \
1394 XFS_IRECLAIM)
1395
1396
1397
1398
1399
1400static bool
1401xfs_blockgc_igrab(
1402 struct xfs_inode *ip)
1403{
1404 struct inode *inode = VFS_I(ip);
1405
1406 ASSERT(rcu_read_lock_held());
1407
1408
1409 spin_lock(&ip->i_flags_lock);
1410 if (!ip->i_ino)
1411 goto out_unlock_noent;
1412
1413 if (ip->i_flags & XFS_BLOCKGC_NOGRAB_IFLAGS)
1414 goto out_unlock_noent;
1415 spin_unlock(&ip->i_flags_lock);
1416
1417
1418 if (xfs_is_shutdown(ip->i_mount))
1419 return false;
1420
1421
1422 if (!igrab(inode))
1423 return false;
1424
1425
1426 return true;
1427
1428out_unlock_noent:
1429 spin_unlock(&ip->i_flags_lock);
1430 return false;
1431}
1432
1433
1434static int
1435xfs_blockgc_scan_inode(
1436 struct xfs_inode *ip,
1437 struct xfs_icwalk *icw)
1438{
1439 unsigned int lockflags = 0;
1440 int error;
1441
1442 error = xfs_inode_free_eofblocks(ip, icw, &lockflags);
1443 if (error)
1444 goto unlock;
1445
1446 error = xfs_inode_free_cowblocks(ip, icw, &lockflags);
1447unlock:
1448 if (lockflags)
1449 xfs_iunlock(ip, lockflags);
1450 xfs_irele(ip);
1451 return error;
1452}
1453
1454
1455void
1456xfs_blockgc_worker(
1457 struct work_struct *work)
1458{
1459 struct xfs_perag *pag = container_of(to_delayed_work(work),
1460 struct xfs_perag, pag_blockgc_work);
1461 struct xfs_mount *mp = pag->pag_mount;
1462 int error;
1463
1464 trace_xfs_blockgc_worker(mp, __return_address);
1465
1466 error = xfs_icwalk_ag(pag, XFS_ICWALK_BLOCKGC, NULL);
1467 if (error)
1468 xfs_info(mp, "AG %u preallocation gc worker failed, err=%d",
1469 pag->pag_agno, error);
1470 xfs_blockgc_queue(pag);
1471}
1472
1473
1474
1475
1476
1477int
1478xfs_blockgc_free_space(
1479 struct xfs_mount *mp,
1480 struct xfs_icwalk *icw)
1481{
1482 int error;
1483
1484 trace_xfs_blockgc_free_space(mp, icw, _RET_IP_);
1485
1486 error = xfs_icwalk(mp, XFS_ICWALK_BLOCKGC, icw);
1487 if (error)
1488 return error;
1489
1490 xfs_inodegc_flush(mp);
1491 return 0;
1492}
1493
1494
1495
1496
1497
1498void
1499xfs_blockgc_flush_all(
1500 struct xfs_mount *mp)
1501{
1502 struct xfs_perag *pag;
1503 xfs_agnumber_t agno;
1504
1505 trace_xfs_blockgc_flush_all(mp, __return_address);
1506
1507
1508
1509
1510
1511
1512 for_each_perag_tag(mp, agno, pag, XFS_ICI_BLOCKGC_TAG)
1513 mod_delayed_work(pag->pag_mount->m_blockgc_wq,
1514 &pag->pag_blockgc_work, 0);
1515
1516 for_each_perag_tag(mp, agno, pag, XFS_ICI_BLOCKGC_TAG)
1517 flush_delayed_work(&pag->pag_blockgc_work);
1518
1519 xfs_inodegc_flush(mp);
1520}
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532int
1533xfs_blockgc_free_dquots(
1534 struct xfs_mount *mp,
1535 struct xfs_dquot *udqp,
1536 struct xfs_dquot *gdqp,
1537 struct xfs_dquot *pdqp,
1538 unsigned int iwalk_flags)
1539{
1540 struct xfs_icwalk icw = {0};
1541 bool do_work = false;
1542
1543 if (!udqp && !gdqp && !pdqp)
1544 return 0;
1545
1546
1547
1548
1549
1550 icw.icw_flags = XFS_ICWALK_FLAG_UNION | iwalk_flags;
1551
1552 if (XFS_IS_UQUOTA_ENFORCED(mp) && udqp && xfs_dquot_lowsp(udqp)) {
1553 icw.icw_uid = make_kuid(mp->m_super->s_user_ns, udqp->q_id);
1554 icw.icw_flags |= XFS_ICWALK_FLAG_UID;
1555 do_work = true;
1556 }
1557
1558 if (XFS_IS_UQUOTA_ENFORCED(mp) && gdqp && xfs_dquot_lowsp(gdqp)) {
1559 icw.icw_gid = make_kgid(mp->m_super->s_user_ns, gdqp->q_id);
1560 icw.icw_flags |= XFS_ICWALK_FLAG_GID;
1561 do_work = true;
1562 }
1563
1564 if (XFS_IS_PQUOTA_ENFORCED(mp) && pdqp && xfs_dquot_lowsp(pdqp)) {
1565 icw.icw_prid = pdqp->q_id;
1566 icw.icw_flags |= XFS_ICWALK_FLAG_PRID;
1567 do_work = true;
1568 }
1569
1570 if (!do_work)
1571 return 0;
1572
1573 return xfs_blockgc_free_space(mp, &icw);
1574}
1575
1576
1577int
1578xfs_blockgc_free_quota(
1579 struct xfs_inode *ip,
1580 unsigned int iwalk_flags)
1581{
1582 return xfs_blockgc_free_dquots(ip->i_mount,
1583 xfs_inode_dquot(ip, XFS_DQTYPE_USER),
1584 xfs_inode_dquot(ip, XFS_DQTYPE_GROUP),
1585 xfs_inode_dquot(ip, XFS_DQTYPE_PROJ), iwalk_flags);
1586}
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596#define XFS_LOOKUP_BATCH 32
1597
1598
1599
1600
1601
1602
1603static inline bool
1604xfs_icwalk_igrab(
1605 enum xfs_icwalk_goal goal,
1606 struct xfs_inode *ip,
1607 struct xfs_icwalk *icw)
1608{
1609 switch (goal) {
1610 case XFS_ICWALK_BLOCKGC:
1611 return xfs_blockgc_igrab(ip);
1612 case XFS_ICWALK_RECLAIM:
1613 return xfs_reclaim_igrab(ip, icw);
1614 default:
1615 return false;
1616 }
1617}
1618
1619
1620
1621
1622
1623static inline int
1624xfs_icwalk_process_inode(
1625 enum xfs_icwalk_goal goal,
1626 struct xfs_inode *ip,
1627 struct xfs_perag *pag,
1628 struct xfs_icwalk *icw)
1629{
1630 int error = 0;
1631
1632 switch (goal) {
1633 case XFS_ICWALK_BLOCKGC:
1634 error = xfs_blockgc_scan_inode(ip, icw);
1635 break;
1636 case XFS_ICWALK_RECLAIM:
1637 xfs_reclaim_inode(ip, pag);
1638 break;
1639 }
1640 return error;
1641}
1642
1643
1644
1645
1646
1647static int
1648xfs_icwalk_ag(
1649 struct xfs_perag *pag,
1650 enum xfs_icwalk_goal goal,
1651 struct xfs_icwalk *icw)
1652{
1653 struct xfs_mount *mp = pag->pag_mount;
1654 uint32_t first_index;
1655 int last_error = 0;
1656 int skipped;
1657 bool done;
1658 int nr_found;
1659
1660restart:
1661 done = false;
1662 skipped = 0;
1663 if (goal == XFS_ICWALK_RECLAIM)
1664 first_index = READ_ONCE(pag->pag_ici_reclaim_cursor);
1665 else
1666 first_index = 0;
1667 nr_found = 0;
1668 do {
1669 struct xfs_inode *batch[XFS_LOOKUP_BATCH];
1670 int error = 0;
1671 int i;
1672
1673 rcu_read_lock();
1674
1675 nr_found = radix_tree_gang_lookup_tag(&pag->pag_ici_root,
1676 (void **) batch, first_index,
1677 XFS_LOOKUP_BATCH, goal);
1678 if (!nr_found) {
1679 done = true;
1680 rcu_read_unlock();
1681 break;
1682 }
1683
1684
1685
1686
1687
1688 for (i = 0; i < nr_found; i++) {
1689 struct xfs_inode *ip = batch[i];
1690
1691 if (done || !xfs_icwalk_igrab(goal, ip, icw))
1692 batch[i] = NULL;
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706 if (XFS_INO_TO_AGNO(mp, ip->i_ino) != pag->pag_agno)
1707 continue;
1708 first_index = XFS_INO_TO_AGINO(mp, ip->i_ino + 1);
1709 if (first_index < XFS_INO_TO_AGINO(mp, ip->i_ino))
1710 done = true;
1711 }
1712
1713
1714 rcu_read_unlock();
1715
1716 for (i = 0; i < nr_found; i++) {
1717 if (!batch[i])
1718 continue;
1719 error = xfs_icwalk_process_inode(goal, batch[i], pag,
1720 icw);
1721 if (error == -EAGAIN) {
1722 skipped++;
1723 continue;
1724 }
1725 if (error && last_error != -EFSCORRUPTED)
1726 last_error = error;
1727 }
1728
1729
1730 if (error == -EFSCORRUPTED)
1731 break;
1732
1733 cond_resched();
1734
1735 if (icw && (icw->icw_flags & XFS_ICWALK_FLAG_SCAN_LIMIT)) {
1736 icw->icw_scan_limit -= XFS_LOOKUP_BATCH;
1737 if (icw->icw_scan_limit <= 0)
1738 break;
1739 }
1740 } while (nr_found && !done);
1741
1742 if (goal == XFS_ICWALK_RECLAIM) {
1743 if (done)
1744 first_index = 0;
1745 WRITE_ONCE(pag->pag_ici_reclaim_cursor, first_index);
1746 }
1747
1748 if (skipped) {
1749 delay(1);
1750 goto restart;
1751 }
1752 return last_error;
1753}
1754
1755
1756static int
1757xfs_icwalk(
1758 struct xfs_mount *mp,
1759 enum xfs_icwalk_goal goal,
1760 struct xfs_icwalk *icw)
1761{
1762 struct xfs_perag *pag;
1763 int error = 0;
1764 int last_error = 0;
1765 xfs_agnumber_t agno;
1766
1767 for_each_perag_tag(mp, agno, pag, goal) {
1768 error = xfs_icwalk_ag(pag, goal, icw);
1769 if (error) {
1770 last_error = error;
1771 if (error == -EFSCORRUPTED) {
1772 xfs_perag_put(pag);
1773 break;
1774 }
1775 }
1776 }
1777 return last_error;
1778 BUILD_BUG_ON(XFS_ICWALK_PRIVATE_FLAGS & XFS_ICWALK_FLAGS_VALID);
1779}
1780
1781#ifdef DEBUG
1782static void
1783xfs_check_delalloc(
1784 struct xfs_inode *ip,
1785 int whichfork)
1786{
1787 struct xfs_ifork *ifp = XFS_IFORK_PTR(ip, whichfork);
1788 struct xfs_bmbt_irec got;
1789 struct xfs_iext_cursor icur;
1790
1791 if (!ifp || !xfs_iext_lookup_extent(ip, ifp, 0, &icur, &got))
1792 return;
1793 do {
1794 if (isnullstartblock(got.br_startblock)) {
1795 xfs_warn(ip->i_mount,
1796 "ino %llx %s fork has delalloc extent at [0x%llx:0x%llx]",
1797 ip->i_ino,
1798 whichfork == XFS_DATA_FORK ? "data" : "cow",
1799 got.br_startoff, got.br_blockcount);
1800 }
1801 } while (xfs_iext_next_extent(ifp, &icur, &got));
1802}
1803#else
1804#define xfs_check_delalloc(ip, whichfork) do { } while (0)
1805#endif
1806
1807
1808static void
1809xfs_inodegc_set_reclaimable(
1810 struct xfs_inode *ip)
1811{
1812 struct xfs_mount *mp = ip->i_mount;
1813 struct xfs_perag *pag;
1814
1815 if (!xfs_is_shutdown(mp) && ip->i_delayed_blks) {
1816 xfs_check_delalloc(ip, XFS_DATA_FORK);
1817 xfs_check_delalloc(ip, XFS_COW_FORK);
1818 ASSERT(0);
1819 }
1820
1821 pag = xfs_perag_get(mp, XFS_INO_TO_AGNO(mp, ip->i_ino));
1822 spin_lock(&pag->pag_ici_lock);
1823 spin_lock(&ip->i_flags_lock);
1824
1825 trace_xfs_inode_set_reclaimable(ip);
1826 ip->i_flags &= ~(XFS_NEED_INACTIVE | XFS_INACTIVATING);
1827 ip->i_flags |= XFS_IRECLAIMABLE;
1828 xfs_perag_set_inode_tag(pag, XFS_INO_TO_AGINO(mp, ip->i_ino),
1829 XFS_ICI_RECLAIM_TAG);
1830
1831 spin_unlock(&ip->i_flags_lock);
1832 spin_unlock(&pag->pag_ici_lock);
1833 xfs_perag_put(pag);
1834}
1835
1836
1837
1838
1839
1840
1841static void
1842xfs_inodegc_inactivate(
1843 struct xfs_inode *ip)
1844{
1845 trace_xfs_inode_inactivating(ip);
1846 xfs_inactive(ip);
1847 xfs_inodegc_set_reclaimable(ip);
1848}
1849
1850void
1851xfs_inodegc_worker(
1852 struct work_struct *work)
1853{
1854 struct xfs_inodegc *gc = container_of(work, struct xfs_inodegc,
1855 work);
1856 struct llist_node *node = llist_del_all(&gc->list);
1857 struct xfs_inode *ip, *n;
1858
1859 WRITE_ONCE(gc->items, 0);
1860
1861 if (!node)
1862 return;
1863
1864 ip = llist_entry(node, struct xfs_inode, i_gclist);
1865 trace_xfs_inodegc_worker(ip->i_mount, READ_ONCE(gc->shrinker_hits));
1866
1867 WRITE_ONCE(gc->shrinker_hits, 0);
1868 llist_for_each_entry_safe(ip, n, node, i_gclist) {
1869 xfs_iflags_set(ip, XFS_INACTIVATING);
1870 xfs_inodegc_inactivate(ip);
1871 }
1872}
1873
1874
1875
1876
1877
1878
1879void
1880xfs_inodegc_flush(
1881 struct xfs_mount *mp)
1882{
1883 struct xfs_inodegc *gc;
1884 int cpu;
1885
1886 if (!xfs_is_inodegc_enabled(mp))
1887 return;
1888
1889 trace_xfs_inodegc_flush(mp, __return_address);
1890
1891 xfs_inodegc_queue_all(mp);
1892
1893 for_each_online_cpu(cpu) {
1894 gc = per_cpu_ptr(mp->m_inodegc, cpu);
1895 flush_work(&gc->work);
1896 }
1897}
1898
1899
1900
1901
1902
1903void
1904xfs_inodegc_stop(
1905 struct xfs_mount *mp)
1906{
1907 struct xfs_inodegc *gc;
1908 int cpu;
1909
1910 if (!xfs_clear_inodegc_enabled(mp))
1911 return;
1912
1913 xfs_inodegc_queue_all(mp);
1914
1915 for_each_online_cpu(cpu) {
1916 gc = per_cpu_ptr(mp->m_inodegc, cpu);
1917 cancel_work_sync(&gc->work);
1918 }
1919 trace_xfs_inodegc_stop(mp, __return_address);
1920}
1921
1922
1923
1924
1925
1926void
1927xfs_inodegc_start(
1928 struct xfs_mount *mp)
1929{
1930 if (xfs_set_inodegc_enabled(mp))
1931 return;
1932
1933 trace_xfs_inodegc_start(mp, __return_address);
1934 xfs_inodegc_queue_all(mp);
1935}
1936
1937#ifdef CONFIG_XFS_RT
1938static inline bool
1939xfs_inodegc_want_queue_rt_file(
1940 struct xfs_inode *ip)
1941{
1942 struct xfs_mount *mp = ip->i_mount;
1943 uint64_t freertx;
1944
1945 if (!XFS_IS_REALTIME_INODE(ip))
1946 return false;
1947
1948 freertx = READ_ONCE(mp->m_sb.sb_frextents);
1949 return freertx < mp->m_low_rtexts[XFS_LOWSP_5_PCNT];
1950}
1951#else
1952# define xfs_inodegc_want_queue_rt_file(ip) (false)
1953#endif
1954
1955
1956
1957
1958
1959
1960
1961
1962static inline bool
1963xfs_inodegc_want_queue_work(
1964 struct xfs_inode *ip,
1965 unsigned int items)
1966{
1967 struct xfs_mount *mp = ip->i_mount;
1968
1969 if (items > mp->m_ino_geo.inodes_per_cluster)
1970 return true;
1971
1972 if (__percpu_counter_compare(&mp->m_fdblocks,
1973 mp->m_low_space[XFS_LOWSP_5_PCNT],
1974 XFS_FDBLOCKS_BATCH) < 0)
1975 return true;
1976
1977 if (xfs_inodegc_want_queue_rt_file(ip))
1978 return true;
1979
1980 if (xfs_inode_near_dquot_enforcement(ip, XFS_DQTYPE_USER))
1981 return true;
1982
1983 if (xfs_inode_near_dquot_enforcement(ip, XFS_DQTYPE_GROUP))
1984 return true;
1985
1986 if (xfs_inode_near_dquot_enforcement(ip, XFS_DQTYPE_PROJ))
1987 return true;
1988
1989 return false;
1990}
1991
1992
1993
1994
1995
1996#define XFS_INODEGC_MAX_BACKLOG (4 * XFS_INODES_PER_CHUNK)
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007static inline bool
2008xfs_inodegc_want_flush_work(
2009 struct xfs_inode *ip,
2010 unsigned int items,
2011 unsigned int shrinker_hits)
2012{
2013 if (current->journal_info)
2014 return false;
2015
2016 if (shrinker_hits > 0)
2017 return true;
2018
2019 if (items > XFS_INODEGC_MAX_BACKLOG)
2020 return true;
2021
2022 return false;
2023}
2024
2025
2026
2027
2028
2029
2030static void
2031xfs_inodegc_queue(
2032 struct xfs_inode *ip)
2033{
2034 struct xfs_mount *mp = ip->i_mount;
2035 struct xfs_inodegc *gc;
2036 int items;
2037 unsigned int shrinker_hits;
2038
2039 trace_xfs_inode_set_need_inactive(ip);
2040 spin_lock(&ip->i_flags_lock);
2041 ip->i_flags |= XFS_NEED_INACTIVE;
2042 spin_unlock(&ip->i_flags_lock);
2043
2044 gc = get_cpu_ptr(mp->m_inodegc);
2045 llist_add(&ip->i_gclist, &gc->list);
2046 items = READ_ONCE(gc->items);
2047 WRITE_ONCE(gc->items, items + 1);
2048 shrinker_hits = READ_ONCE(gc->shrinker_hits);
2049 put_cpu_ptr(gc);
2050
2051 if (!xfs_is_inodegc_enabled(mp))
2052 return;
2053
2054 if (xfs_inodegc_want_queue_work(ip, items)) {
2055 trace_xfs_inodegc_queue(mp, __return_address);
2056 queue_work(mp->m_inodegc_wq, &gc->work);
2057 }
2058
2059 if (xfs_inodegc_want_flush_work(ip, items, shrinker_hits)) {
2060 trace_xfs_inodegc_throttle(mp, __return_address);
2061 flush_work(&gc->work);
2062 }
2063}
2064
2065
2066
2067
2068void
2069xfs_inodegc_cpu_dead(
2070 struct xfs_mount *mp,
2071 unsigned int dead_cpu)
2072{
2073 struct xfs_inodegc *dead_gc, *gc;
2074 struct llist_node *first, *last;
2075 unsigned int count = 0;
2076
2077 dead_gc = per_cpu_ptr(mp->m_inodegc, dead_cpu);
2078 cancel_work_sync(&dead_gc->work);
2079
2080 if (llist_empty(&dead_gc->list))
2081 return;
2082
2083 first = dead_gc->list.first;
2084 last = first;
2085 while (last->next) {
2086 last = last->next;
2087 count++;
2088 }
2089 dead_gc->list.first = NULL;
2090 dead_gc->items = 0;
2091
2092
2093 gc = get_cpu_ptr(mp->m_inodegc);
2094 llist_add_batch(first, last, &gc->list);
2095 count += READ_ONCE(gc->items);
2096 WRITE_ONCE(gc->items, count);
2097 put_cpu_ptr(gc);
2098
2099 if (xfs_is_inodegc_enabled(mp)) {
2100 trace_xfs_inodegc_queue(mp, __return_address);
2101 queue_work(mp->m_inodegc_wq, &gc->work);
2102 }
2103}
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115void
2116xfs_inode_mark_reclaimable(
2117 struct xfs_inode *ip)
2118{
2119 struct xfs_mount *mp = ip->i_mount;
2120 bool need_inactive;
2121
2122 XFS_STATS_INC(mp, vn_reclaim);
2123
2124
2125
2126
2127 ASSERT_ALWAYS(!xfs_iflags_test(ip, XFS_ALL_IRECLAIM_FLAGS));
2128
2129 need_inactive = xfs_inode_needs_inactive(ip);
2130 if (need_inactive) {
2131 xfs_inodegc_queue(ip);
2132 return;
2133 }
2134
2135
2136 xfs_qm_dqdetach(ip);
2137 xfs_inodegc_set_reclaimable(ip);
2138}
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150#define XFS_INODEGC_SHRINKER_COUNT (1UL << DEF_PRIORITY)
2151#define XFS_INODEGC_SHRINKER_BATCH ((XFS_INODEGC_SHRINKER_COUNT / 2) + 1)
2152
2153static unsigned long
2154xfs_inodegc_shrinker_count(
2155 struct shrinker *shrink,
2156 struct shrink_control *sc)
2157{
2158 struct xfs_mount *mp = container_of(shrink, struct xfs_mount,
2159 m_inodegc_shrinker);
2160 struct xfs_inodegc *gc;
2161 int cpu;
2162
2163 if (!xfs_is_inodegc_enabled(mp))
2164 return 0;
2165
2166 for_each_online_cpu(cpu) {
2167 gc = per_cpu_ptr(mp->m_inodegc, cpu);
2168 if (!llist_empty(&gc->list))
2169 return XFS_INODEGC_SHRINKER_COUNT;
2170 }
2171
2172 return 0;
2173}
2174
2175static unsigned long
2176xfs_inodegc_shrinker_scan(
2177 struct shrinker *shrink,
2178 struct shrink_control *sc)
2179{
2180 struct xfs_mount *mp = container_of(shrink, struct xfs_mount,
2181 m_inodegc_shrinker);
2182 struct xfs_inodegc *gc;
2183 int cpu;
2184 bool no_items = true;
2185
2186 if (!xfs_is_inodegc_enabled(mp))
2187 return SHRINK_STOP;
2188
2189 trace_xfs_inodegc_shrinker_scan(mp, sc, __return_address);
2190
2191 for_each_online_cpu(cpu) {
2192 gc = per_cpu_ptr(mp->m_inodegc, cpu);
2193 if (!llist_empty(&gc->list)) {
2194 unsigned int h = READ_ONCE(gc->shrinker_hits);
2195
2196 WRITE_ONCE(gc->shrinker_hits, h + 1);
2197 queue_work_on(cpu, mp->m_inodegc_wq, &gc->work);
2198 no_items = false;
2199 }
2200 }
2201
2202
2203
2204
2205
2206 if (no_items)
2207 return LONG_MAX;
2208
2209 return SHRINK_STOP;
2210}
2211
2212
2213int
2214xfs_inodegc_register_shrinker(
2215 struct xfs_mount *mp)
2216{
2217 struct shrinker *shrink = &mp->m_inodegc_shrinker;
2218
2219 shrink->count_objects = xfs_inodegc_shrinker_count;
2220 shrink->scan_objects = xfs_inodegc_shrinker_scan;
2221 shrink->seeks = 0;
2222 shrink->flags = SHRINKER_NONSLAB;
2223 shrink->batch = XFS_INODEGC_SHRINKER_BATCH;
2224
2225 return register_shrinker(shrink);
2226}
2227