1
2
3
4
5
6#include "xfs.h"
7#include "xfs_fs.h"
8#include "xfs_shared.h"
9#include "xfs_format.h"
10#include "xfs_log_format.h"
11#include "xfs_trans_resv.h"
12#include "xfs_mount.h"
13#include "xfs_inode.h"
14#include "xfs_btree.h"
15#include "xfs_ialloc.h"
16#include "xfs_ialloc_btree.h"
17#include "xfs_iwalk.h"
18#include "xfs_error.h"
19#include "xfs_trace.h"
20#include "xfs_icache.h"
21#include "xfs_health.h"
22#include "xfs_trans.h"
23#include "xfs_pwork.h"
24#include "xfs_ag.h"
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49struct xfs_iwalk_ag {
50
51 struct xfs_pwork pwork;
52
53 struct xfs_mount *mp;
54 struct xfs_trans *tp;
55 struct xfs_perag *pag;
56
57
58 xfs_ino_t startino;
59
60
61 xfs_ino_t lastino;
62
63
64 struct xfs_inobt_rec_incore *recs;
65
66
67 unsigned int sz_recs;
68
69
70 unsigned int nr_recs;
71
72
73 xfs_iwalk_fn iwalk_fn;
74 xfs_inobt_walk_fn inobt_walk_fn;
75 void *data;
76
77
78
79
80
81
82 unsigned int trim_start:1;
83
84
85 unsigned int skip_empty:1;
86
87
88 unsigned int drop_trans:1;
89};
90
91
92
93
94
95STATIC void
96xfs_iwalk_ichunk_ra(
97 struct xfs_mount *mp,
98 struct xfs_perag *pag,
99 struct xfs_inobt_rec_incore *irec)
100{
101 struct xfs_ino_geometry *igeo = M_IGEO(mp);
102 xfs_agblock_t agbno;
103 struct blk_plug plug;
104 int i;
105
106 agbno = XFS_AGINO_TO_AGBNO(mp, irec->ir_startino);
107
108 blk_start_plug(&plug);
109 for (i = 0; i < XFS_INODES_PER_CHUNK; i += igeo->inodes_per_cluster) {
110 xfs_inofree_t imask;
111
112 imask = xfs_inobt_maskn(i, igeo->inodes_per_cluster);
113 if (imask & ~irec->ir_free) {
114 xfs_btree_reada_bufs(mp, pag->pag_agno, agbno,
115 igeo->blocks_per_cluster,
116 &xfs_inode_buf_ops);
117 }
118 agbno += igeo->blocks_per_cluster;
119 }
120 blk_finish_plug(&plug);
121}
122
123
124
125
126
127
128STATIC void
129xfs_iwalk_adjust_start(
130 xfs_agino_t agino,
131 struct xfs_inobt_rec_incore *irec)
132{
133 int idx;
134 int i;
135
136 idx = agino - irec->ir_startino;
137
138
139
140
141
142
143 for (i = 0; i < idx; i++) {
144 if (XFS_INOBT_MASK(i) & ~irec->ir_free)
145 irec->ir_freecount++;
146 }
147
148 irec->ir_free |= xfs_inobt_maskn(0, idx);
149}
150
151
152STATIC int
153xfs_iwalk_alloc(
154 struct xfs_iwalk_ag *iwag)
155{
156 size_t size;
157
158 ASSERT(iwag->recs == NULL);
159 iwag->nr_recs = 0;
160
161
162 size = iwag->sz_recs * sizeof(struct xfs_inobt_rec_incore);
163 iwag->recs = kmem_alloc(size, KM_MAYFAIL);
164 if (iwag->recs == NULL)
165 return -ENOMEM;
166
167 return 0;
168}
169
170
171STATIC void
172xfs_iwalk_free(
173 struct xfs_iwalk_ag *iwag)
174{
175 kmem_free(iwag->recs);
176 iwag->recs = NULL;
177}
178
179
180STATIC int
181xfs_iwalk_ag_recs(
182 struct xfs_iwalk_ag *iwag)
183{
184 struct xfs_mount *mp = iwag->mp;
185 struct xfs_trans *tp = iwag->tp;
186 struct xfs_perag *pag = iwag->pag;
187 xfs_ino_t ino;
188 unsigned int i, j;
189 int error;
190
191 for (i = 0; i < iwag->nr_recs; i++) {
192 struct xfs_inobt_rec_incore *irec = &iwag->recs[i];
193
194 trace_xfs_iwalk_ag_rec(mp, pag->pag_agno, irec);
195
196 if (xfs_pwork_want_abort(&iwag->pwork))
197 return 0;
198
199 if (iwag->inobt_walk_fn) {
200 error = iwag->inobt_walk_fn(mp, tp, pag->pag_agno, irec,
201 iwag->data);
202 if (error)
203 return error;
204 }
205
206 if (!iwag->iwalk_fn)
207 continue;
208
209 for (j = 0; j < XFS_INODES_PER_CHUNK; j++) {
210 if (xfs_pwork_want_abort(&iwag->pwork))
211 return 0;
212
213
214 if (XFS_INOBT_MASK(j) & irec->ir_free)
215 continue;
216
217
218 ino = XFS_AGINO_TO_INO(mp, pag->pag_agno,
219 irec->ir_startino + j);
220 error = iwag->iwalk_fn(mp, tp, ino, iwag->data);
221 if (error)
222 return error;
223 }
224 }
225
226 return 0;
227}
228
229
230static inline void
231xfs_iwalk_del_inobt(
232 struct xfs_trans *tp,
233 struct xfs_btree_cur **curpp,
234 struct xfs_buf **agi_bpp,
235 int error)
236{
237 if (*curpp) {
238 xfs_btree_del_cursor(*curpp, error);
239 *curpp = NULL;
240 }
241 if (*agi_bpp) {
242 xfs_trans_brelse(tp, *agi_bpp);
243 *agi_bpp = NULL;
244 }
245}
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262STATIC int
263xfs_iwalk_ag_start(
264 struct xfs_iwalk_ag *iwag,
265 xfs_agino_t agino,
266 struct xfs_btree_cur **curpp,
267 struct xfs_buf **agi_bpp,
268 int *has_more)
269{
270 struct xfs_mount *mp = iwag->mp;
271 struct xfs_trans *tp = iwag->tp;
272 struct xfs_perag *pag = iwag->pag;
273 struct xfs_inobt_rec_incore *irec;
274 int error;
275
276
277 iwag->nr_recs = 0;
278 error = xfs_inobt_cur(mp, tp, pag, XFS_BTNUM_INO, curpp, agi_bpp);
279 if (error)
280 return error;
281
282
283 if (agino == 0)
284 return xfs_inobt_lookup(*curpp, 0, XFS_LOOKUP_GE, has_more);
285
286
287
288
289
290
291
292
293 error = xfs_inobt_lookup(*curpp, agino, XFS_LOOKUP_LE, has_more);
294 if (error)
295 return error;
296
297
298
299
300
301 if (!*has_more)
302 goto out_advance;
303
304
305 irec = &iwag->recs[iwag->nr_recs];
306 error = xfs_inobt_get_rec(*curpp, irec, has_more);
307 if (error)
308 return error;
309 if (XFS_IS_CORRUPT(mp, *has_more != 1))
310 return -EFSCORRUPTED;
311
312 iwag->lastino = XFS_AGINO_TO_INO(mp, pag->pag_agno,
313 irec->ir_startino + XFS_INODES_PER_CHUNK - 1);
314
315
316
317
318
319 if (irec->ir_startino + XFS_INODES_PER_CHUNK <= agino)
320 goto out_advance;
321
322
323
324
325
326 if (iwag->trim_start)
327 xfs_iwalk_adjust_start(agino, irec);
328
329
330
331
332
333
334
335 iwag->nr_recs++;
336 ASSERT(iwag->nr_recs < iwag->sz_recs);
337
338out_advance:
339 return xfs_btree_increment(*curpp, 0, has_more);
340}
341
342
343
344
345
346
347
348
349
350STATIC int
351xfs_iwalk_run_callbacks(
352 struct xfs_iwalk_ag *iwag,
353 struct xfs_btree_cur **curpp,
354 struct xfs_buf **agi_bpp,
355 int *has_more)
356{
357 struct xfs_mount *mp = iwag->mp;
358 struct xfs_inobt_rec_incore *irec;
359 xfs_agino_t next_agino;
360 int error;
361
362 next_agino = XFS_INO_TO_AGINO(mp, iwag->lastino) + 1;
363
364 ASSERT(iwag->nr_recs > 0);
365
366
367 xfs_iwalk_del_inobt(iwag->tp, curpp, agi_bpp, 0);
368 irec = &iwag->recs[iwag->nr_recs - 1];
369 ASSERT(next_agino >= irec->ir_startino + XFS_INODES_PER_CHUNK);
370
371 if (iwag->drop_trans) {
372 xfs_trans_cancel(iwag->tp);
373 iwag->tp = NULL;
374 }
375
376 error = xfs_iwalk_ag_recs(iwag);
377 if (error)
378 return error;
379
380
381 iwag->nr_recs = 0;
382
383 if (!has_more)
384 return 0;
385
386 if (iwag->drop_trans) {
387 error = xfs_trans_alloc_empty(mp, &iwag->tp);
388 if (error)
389 return error;
390 }
391
392
393 error = xfs_inobt_cur(mp, iwag->tp, iwag->pag, XFS_BTNUM_INO, curpp,
394 agi_bpp);
395 if (error)
396 return error;
397
398 return xfs_inobt_lookup(*curpp, next_agino, XFS_LOOKUP_GE, has_more);
399}
400
401
402STATIC int
403xfs_iwalk_ag(
404 struct xfs_iwalk_ag *iwag)
405{
406 struct xfs_mount *mp = iwag->mp;
407 struct xfs_perag *pag = iwag->pag;
408 struct xfs_buf *agi_bp = NULL;
409 struct xfs_btree_cur *cur = NULL;
410 xfs_agino_t agino;
411 int has_more;
412 int error = 0;
413
414
415 ASSERT(pag->pag_agno == XFS_INO_TO_AGNO(mp, iwag->startino));
416 agino = XFS_INO_TO_AGINO(mp, iwag->startino);
417 error = xfs_iwalk_ag_start(iwag, agino, &cur, &agi_bp, &has_more);
418
419 while (!error && has_more) {
420 struct xfs_inobt_rec_incore *irec;
421 xfs_ino_t rec_fsino;
422
423 cond_resched();
424 if (xfs_pwork_want_abort(&iwag->pwork))
425 goto out;
426
427
428 irec = &iwag->recs[iwag->nr_recs];
429 error = xfs_inobt_get_rec(cur, irec, &has_more);
430 if (error || !has_more)
431 break;
432
433
434 rec_fsino = XFS_AGINO_TO_INO(mp, pag->pag_agno, irec->ir_startino);
435 if (iwag->lastino != NULLFSINO &&
436 XFS_IS_CORRUPT(mp, iwag->lastino >= rec_fsino)) {
437 error = -EFSCORRUPTED;
438 goto out;
439 }
440 iwag->lastino = rec_fsino + XFS_INODES_PER_CHUNK - 1;
441
442
443 if (iwag->skip_empty && irec->ir_freecount == irec->ir_count) {
444 error = xfs_btree_increment(cur, 0, &has_more);
445 if (error)
446 break;
447 continue;
448 }
449
450
451
452
453
454 if (iwag->iwalk_fn)
455 xfs_iwalk_ichunk_ra(mp, pag, irec);
456
457
458
459
460
461 if (++iwag->nr_recs < iwag->sz_recs) {
462 error = xfs_btree_increment(cur, 0, &has_more);
463 if (error || !has_more)
464 break;
465 continue;
466 }
467
468
469
470
471
472
473
474 ASSERT(has_more);
475 error = xfs_iwalk_run_callbacks(iwag, &cur, &agi_bp, &has_more);
476 }
477
478 if (iwag->nr_recs == 0 || error)
479 goto out;
480
481
482 error = xfs_iwalk_run_callbacks(iwag, &cur, &agi_bp, &has_more);
483
484out:
485 xfs_iwalk_del_inobt(iwag->tp, &cur, &agi_bp, error);
486 return error;
487}
488
489
490
491
492
493
494#define IWALK_MAX_INODE_PREFETCH (2048U)
495
496
497
498
499
500
501static inline unsigned int
502xfs_iwalk_prefetch(
503 unsigned int inodes)
504{
505 unsigned int inobt_records;
506
507
508
509
510
511
512
513 if (inodes == 0)
514 inodes = IWALK_MAX_INODE_PREFETCH;
515 inodes = min(inodes, IWALK_MAX_INODE_PREFETCH);
516
517
518 inodes = round_up(inodes, XFS_INODES_PER_CHUNK);
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533 inobt_records = (inodes * 5) / (4 * XFS_INODES_PER_CHUNK);
534
535
536
537
538
539
540 return max(inobt_records, 2U);
541}
542
543
544
545
546
547
548
549int
550xfs_iwalk(
551 struct xfs_mount *mp,
552 struct xfs_trans *tp,
553 xfs_ino_t startino,
554 unsigned int flags,
555 xfs_iwalk_fn iwalk_fn,
556 unsigned int inode_records,
557 void *data)
558{
559 struct xfs_iwalk_ag iwag = {
560 .mp = mp,
561 .tp = tp,
562 .iwalk_fn = iwalk_fn,
563 .data = data,
564 .startino = startino,
565 .sz_recs = xfs_iwalk_prefetch(inode_records),
566 .trim_start = 1,
567 .skip_empty = 1,
568 .pwork = XFS_PWORK_SINGLE_THREADED,
569 .lastino = NULLFSINO,
570 };
571 struct xfs_perag *pag;
572 xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
573 int error;
574
575 ASSERT(agno < mp->m_sb.sb_agcount);
576 ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL));
577
578 error = xfs_iwalk_alloc(&iwag);
579 if (error)
580 return error;
581
582 for_each_perag_from(mp, agno, pag) {
583 iwag.pag = pag;
584 error = xfs_iwalk_ag(&iwag);
585 if (error)
586 break;
587 iwag.startino = XFS_AGINO_TO_INO(mp, agno + 1, 0);
588 if (flags & XFS_INOBT_WALK_SAME_AG)
589 break;
590 iwag.pag = NULL;
591 }
592
593 if (iwag.pag)
594 xfs_perag_put(pag);
595 xfs_iwalk_free(&iwag);
596 return error;
597}
598
599
600static int
601xfs_iwalk_ag_work(
602 struct xfs_mount *mp,
603 struct xfs_pwork *pwork)
604{
605 struct xfs_iwalk_ag *iwag;
606 int error = 0;
607
608 iwag = container_of(pwork, struct xfs_iwalk_ag, pwork);
609 if (xfs_pwork_want_abort(pwork))
610 goto out;
611
612 error = xfs_iwalk_alloc(iwag);
613 if (error)
614 goto out;
615
616
617
618
619 error = xfs_trans_alloc_empty(mp, &iwag->tp);
620 if (error)
621 goto out;
622 iwag->drop_trans = 1;
623
624 error = xfs_iwalk_ag(iwag);
625 if (iwag->tp)
626 xfs_trans_cancel(iwag->tp);
627 xfs_iwalk_free(iwag);
628out:
629 xfs_perag_put(iwag->pag);
630 kmem_free(iwag);
631 return error;
632}
633
634
635
636
637
638int
639xfs_iwalk_threaded(
640 struct xfs_mount *mp,
641 xfs_ino_t startino,
642 unsigned int flags,
643 xfs_iwalk_fn iwalk_fn,
644 unsigned int inode_records,
645 bool polled,
646 void *data)
647{
648 struct xfs_pwork_ctl pctl;
649 struct xfs_perag *pag;
650 xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
651 int error;
652
653 ASSERT(agno < mp->m_sb.sb_agcount);
654 ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL));
655
656 error = xfs_pwork_init(mp, &pctl, xfs_iwalk_ag_work, "xfs_iwalk");
657 if (error)
658 return error;
659
660 for_each_perag_from(mp, agno, pag) {
661 struct xfs_iwalk_ag *iwag;
662
663 if (xfs_pwork_ctl_want_abort(&pctl))
664 break;
665
666 iwag = kmem_zalloc(sizeof(struct xfs_iwalk_ag), 0);
667 iwag->mp = mp;
668
669
670
671
672
673 atomic_inc(&pag->pag_ref);
674 iwag->pag = pag;
675 iwag->iwalk_fn = iwalk_fn;
676 iwag->data = data;
677 iwag->startino = startino;
678 iwag->sz_recs = xfs_iwalk_prefetch(inode_records);
679 iwag->lastino = NULLFSINO;
680 xfs_pwork_queue(&pctl, &iwag->pwork);
681 startino = XFS_AGINO_TO_INO(mp, pag->pag_agno + 1, 0);
682 if (flags & XFS_INOBT_WALK_SAME_AG)
683 break;
684 }
685 if (pag)
686 xfs_perag_put(pag);
687 if (polled)
688 xfs_pwork_poll(&pctl);
689 return xfs_pwork_destroy(&pctl);
690}
691
692
693
694
695
696
697
698#define MAX_INOBT_WALK_PREFETCH \
699 (PAGE_SIZE / sizeof(struct xfs_inobt_rec_incore))
700
701
702
703
704
705static inline unsigned int
706xfs_inobt_walk_prefetch(
707 unsigned int inobt_records)
708{
709
710
711
712
713 if (inobt_records == 0)
714 inobt_records = MAX_INOBT_WALK_PREFETCH;
715
716
717
718
719
720
721 inobt_records = max(inobt_records, 2U);
722
723
724
725
726
727 return min_t(unsigned int, inobt_records, MAX_INOBT_WALK_PREFETCH);
728}
729
730
731
732
733
734
735
736int
737xfs_inobt_walk(
738 struct xfs_mount *mp,
739 struct xfs_trans *tp,
740 xfs_ino_t startino,
741 unsigned int flags,
742 xfs_inobt_walk_fn inobt_walk_fn,
743 unsigned int inobt_records,
744 void *data)
745{
746 struct xfs_iwalk_ag iwag = {
747 .mp = mp,
748 .tp = tp,
749 .inobt_walk_fn = inobt_walk_fn,
750 .data = data,
751 .startino = startino,
752 .sz_recs = xfs_inobt_walk_prefetch(inobt_records),
753 .pwork = XFS_PWORK_SINGLE_THREADED,
754 .lastino = NULLFSINO,
755 };
756 struct xfs_perag *pag;
757 xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
758 int error;
759
760 ASSERT(agno < mp->m_sb.sb_agcount);
761 ASSERT(!(flags & ~XFS_INOBT_WALK_FLAGS_ALL));
762
763 error = xfs_iwalk_alloc(&iwag);
764 if (error)
765 return error;
766
767 for_each_perag_from(mp, agno, pag) {
768 iwag.pag = pag;
769 error = xfs_iwalk_ag(&iwag);
770 if (error)
771 break;
772 iwag.startino = XFS_AGINO_TO_INO(mp, pag->pag_agno + 1, 0);
773 if (flags & XFS_INOBT_WALK_SAME_AG)
774 break;
775 iwag.pag = NULL;
776 }
777
778 if (iwag.pag)
779 xfs_perag_put(pag);
780 xfs_iwalk_free(&iwag);
781 return error;
782}
783