1
2
3
4
5
6
7
8
9
10
11#include <linux/blkdev.h>
12#include <linux/backing-dev.h>
13
14
15#define NULL_SEGNO ((unsigned int)(~0))
16#define NULL_SECNO ((unsigned int)(~0))
17
18#define DEF_RECLAIM_PREFREE_SEGMENTS 5
19#define DEF_MAX_RECLAIM_PREFREE_SEGMENTS 4096
20
21#define F2FS_MIN_SEGMENTS 9
22
23
24#define GET_L2R_SEGNO(free_i, segno) ((segno) - (free_i)->start_segno)
25#define GET_R2L_SEGNO(free_i, segno) ((segno) + (free_i)->start_segno)
26
27#define IS_DATASEG(t) ((t) <= CURSEG_COLD_DATA)
28#define IS_NODESEG(t) ((t) >= CURSEG_HOT_NODE)
29
30#define IS_HOT(t) ((t) == CURSEG_HOT_NODE || (t) == CURSEG_HOT_DATA)
31#define IS_WARM(t) ((t) == CURSEG_WARM_NODE || (t) == CURSEG_WARM_DATA)
32#define IS_COLD(t) ((t) == CURSEG_COLD_NODE || (t) == CURSEG_COLD_DATA)
33
34#define IS_CURSEG(sbi, seg) \
35 (((seg) == CURSEG_I(sbi, CURSEG_HOT_DATA)->segno) || \
36 ((seg) == CURSEG_I(sbi, CURSEG_WARM_DATA)->segno) || \
37 ((seg) == CURSEG_I(sbi, CURSEG_COLD_DATA)->segno) || \
38 ((seg) == CURSEG_I(sbi, CURSEG_HOT_NODE)->segno) || \
39 ((seg) == CURSEG_I(sbi, CURSEG_WARM_NODE)->segno) || \
40 ((seg) == CURSEG_I(sbi, CURSEG_COLD_NODE)->segno))
41
42#define IS_CURSEC(sbi, secno) \
43 (((secno) == CURSEG_I(sbi, CURSEG_HOT_DATA)->segno / \
44 (sbi)->segs_per_sec) || \
45 ((secno) == CURSEG_I(sbi, CURSEG_WARM_DATA)->segno / \
46 (sbi)->segs_per_sec) || \
47 ((secno) == CURSEG_I(sbi, CURSEG_COLD_DATA)->segno / \
48 (sbi)->segs_per_sec) || \
49 ((secno) == CURSEG_I(sbi, CURSEG_HOT_NODE)->segno / \
50 (sbi)->segs_per_sec) || \
51 ((secno) == CURSEG_I(sbi, CURSEG_WARM_NODE)->segno / \
52 (sbi)->segs_per_sec) || \
53 ((secno) == CURSEG_I(sbi, CURSEG_COLD_NODE)->segno / \
54 (sbi)->segs_per_sec)) \
55
56#define MAIN_BLKADDR(sbi) (SM_I(sbi)->main_blkaddr)
57#define SEG0_BLKADDR(sbi) (SM_I(sbi)->seg0_blkaddr)
58
59#define MAIN_SEGS(sbi) (SM_I(sbi)->main_segments)
60#define MAIN_SECS(sbi) ((sbi)->total_sections)
61
62#define TOTAL_SEGS(sbi) (SM_I(sbi)->segment_count)
63#define TOTAL_BLKS(sbi) (TOTAL_SEGS(sbi) << (sbi)->log_blocks_per_seg)
64
65#define MAX_BLKADDR(sbi) (SEG0_BLKADDR(sbi) + TOTAL_BLKS(sbi))
66#define SEGMENT_SIZE(sbi) (1ULL << ((sbi)->log_blocksize + \
67 (sbi)->log_blocks_per_seg))
68
69#define START_BLOCK(sbi, segno) (SEG0_BLKADDR(sbi) + \
70 (GET_R2L_SEGNO(FREE_I(sbi), segno) << (sbi)->log_blocks_per_seg))
71
72#define NEXT_FREE_BLKADDR(sbi, curseg) \
73 (START_BLOCK(sbi, (curseg)->segno) + (curseg)->next_blkoff)
74
75#define GET_SEGOFF_FROM_SEG0(sbi, blk_addr) ((blk_addr) - SEG0_BLKADDR(sbi))
76#define GET_SEGNO_FROM_SEG0(sbi, blk_addr) \
77 (GET_SEGOFF_FROM_SEG0(sbi, blk_addr) >> (sbi)->log_blocks_per_seg)
78#define GET_BLKOFF_FROM_SEG0(sbi, blk_addr) \
79 (GET_SEGOFF_FROM_SEG0(sbi, blk_addr) & ((sbi)->blocks_per_seg - 1))
80
81#define GET_SEGNO(sbi, blk_addr) \
82 ((((blk_addr) == NULL_ADDR) || ((blk_addr) == NEW_ADDR)) ? \
83 NULL_SEGNO : GET_L2R_SEGNO(FREE_I(sbi), \
84 GET_SEGNO_FROM_SEG0(sbi, blk_addr)))
85#define BLKS_PER_SEC(sbi) \
86 ((sbi)->segs_per_sec * (sbi)->blocks_per_seg)
87#define GET_SEC_FROM_SEG(sbi, segno) \
88 ((segno) / (sbi)->segs_per_sec)
89#define GET_SEG_FROM_SEC(sbi, secno) \
90 ((secno) * (sbi)->segs_per_sec)
91#define GET_ZONE_FROM_SEC(sbi, secno) \
92 ((secno) / (sbi)->secs_per_zone)
93#define GET_ZONE_FROM_SEG(sbi, segno) \
94 GET_ZONE_FROM_SEC(sbi, GET_SEC_FROM_SEG(sbi, segno))
95
96#define GET_SUM_BLOCK(sbi, segno) \
97 ((sbi)->sm_info->ssa_blkaddr + (segno))
98
99#define GET_SUM_TYPE(footer) ((footer)->entry_type)
100#define SET_SUM_TYPE(footer, type) ((footer)->entry_type = (type))
101
102#define SIT_ENTRY_OFFSET(sit_i, segno) \
103 ((segno) % (sit_i)->sents_per_block)
104#define SIT_BLOCK_OFFSET(segno) \
105 ((segno) / SIT_ENTRY_PER_BLOCK)
106#define START_SEGNO(segno) \
107 (SIT_BLOCK_OFFSET(segno) * SIT_ENTRY_PER_BLOCK)
108#define SIT_BLK_CNT(sbi) \
109 ((MAIN_SEGS(sbi) + SIT_ENTRY_PER_BLOCK - 1) / SIT_ENTRY_PER_BLOCK)
110#define f2fs_bitmap_size(nr) \
111 (BITS_TO_LONGS(nr) * sizeof(unsigned long))
112
113#define SECTOR_FROM_BLOCK(blk_addr) \
114 (((sector_t)blk_addr) << F2FS_LOG_SECTORS_PER_BLOCK)
115#define SECTOR_TO_BLOCK(sectors) \
116 ((sectors) >> F2FS_LOG_SECTORS_PER_BLOCK)
117
118
119
120
121
122
123enum {
124 ALLOC_RIGHT = 0,
125 ALLOC_LEFT
126};
127
128
129
130
131
132
133enum {
134 LFS = 0,
135 SSR
136};
137
138
139
140
141
142
143enum {
144 GC_CB = 0,
145 GC_GREEDY,
146 ALLOC_NEXT,
147 FLUSH_DEVICE,
148 MAX_GC_POLICY,
149};
150
151
152
153
154
155
156enum {
157 BG_GC = 0,
158 FG_GC,
159 FORCE_FG_GC,
160};
161
162
163struct victim_sel_policy {
164 int alloc_mode;
165 int gc_mode;
166 unsigned long *dirty_segmap;
167 unsigned int max_search;
168 unsigned int offset;
169 unsigned int ofs_unit;
170 unsigned int min_cost;
171 unsigned int min_segno;
172};
173
174struct seg_entry {
175 unsigned int type:6;
176 unsigned int valid_blocks:10;
177 unsigned int ckpt_valid_blocks:10;
178 unsigned int padding:6;
179 unsigned char *cur_valid_map;
180#ifdef CONFIG_F2FS_CHECK_FS
181 unsigned char *cur_valid_map_mir;
182#endif
183
184
185
186
187 unsigned char *ckpt_valid_map;
188 unsigned char *discard_map;
189 unsigned long long mtime;
190};
191
192struct sec_entry {
193 unsigned int valid_blocks;
194};
195
196struct segment_allocation {
197 void (*allocate_segment)(struct f2fs_sb_info *, int, bool);
198};
199
200
201
202
203
204#define ATOMIC_WRITTEN_PAGE ((unsigned long)-1)
205#define DUMMY_WRITTEN_PAGE ((unsigned long)-2)
206
207#define IS_ATOMIC_WRITTEN_PAGE(page) \
208 (page_private(page) == (unsigned long)ATOMIC_WRITTEN_PAGE)
209#define IS_DUMMY_WRITTEN_PAGE(page) \
210 (page_private(page) == (unsigned long)DUMMY_WRITTEN_PAGE)
211
212struct inmem_pages {
213 struct list_head list;
214 struct page *page;
215 block_t old_addr;
216};
217
218struct sit_info {
219 const struct segment_allocation *s_ops;
220
221 block_t sit_base_addr;
222 block_t sit_blocks;
223 block_t written_valid_blocks;
224 char *sit_bitmap;
225#ifdef CONFIG_F2FS_CHECK_FS
226 char *sit_bitmap_mir;
227#endif
228 unsigned int bitmap_size;
229
230 unsigned long *tmp_map;
231 unsigned long *dirty_sentries_bitmap;
232 unsigned int dirty_sentries;
233 unsigned int sents_per_block;
234 struct rw_semaphore sentry_lock;
235 struct seg_entry *sentries;
236 struct sec_entry *sec_entries;
237
238
239 unsigned long long elapsed_time;
240 unsigned long long mounted_time;
241 unsigned long long min_mtime;
242 unsigned long long max_mtime;
243
244 unsigned int last_victim[MAX_GC_POLICY];
245};
246
247struct free_segmap_info {
248 unsigned int start_segno;
249 unsigned int free_segments;
250 unsigned int free_sections;
251 spinlock_t segmap_lock;
252 unsigned long *free_segmap;
253 unsigned long *free_secmap;
254};
255
256
257enum dirty_type {
258 DIRTY_HOT_DATA,
259 DIRTY_WARM_DATA,
260 DIRTY_COLD_DATA,
261 DIRTY_HOT_NODE,
262 DIRTY_WARM_NODE,
263 DIRTY_COLD_NODE,
264 DIRTY,
265 PRE,
266 NR_DIRTY_TYPE
267};
268
269struct dirty_seglist_info {
270 const struct victim_selection *v_ops;
271 unsigned long *dirty_segmap[NR_DIRTY_TYPE];
272 struct mutex seglist_lock;
273 int nr_dirty[NR_DIRTY_TYPE];
274 unsigned long *victim_secmap;
275};
276
277
278struct victim_selection {
279 int (*get_victim)(struct f2fs_sb_info *, unsigned int *,
280 int, int, char);
281};
282
283
284struct curseg_info {
285 struct mutex curseg_mutex;
286 struct f2fs_summary_block *sum_blk;
287 struct rw_semaphore journal_rwsem;
288 struct f2fs_journal *journal;
289 unsigned char alloc_type;
290 unsigned int segno;
291 unsigned short next_blkoff;
292 unsigned int zone;
293 unsigned int next_segno;
294};
295
296struct sit_entry_set {
297 struct list_head set_list;
298 unsigned int start_segno;
299 unsigned int entry_cnt;
300};
301
302
303
304
305static inline struct curseg_info *CURSEG_I(struct f2fs_sb_info *sbi, int type)
306{
307 return (struct curseg_info *)(SM_I(sbi)->curseg_array + type);
308}
309
310static inline struct seg_entry *get_seg_entry(struct f2fs_sb_info *sbi,
311 unsigned int segno)
312{
313 struct sit_info *sit_i = SIT_I(sbi);
314 return &sit_i->sentries[segno];
315}
316
317static inline struct sec_entry *get_sec_entry(struct f2fs_sb_info *sbi,
318 unsigned int segno)
319{
320 struct sit_info *sit_i = SIT_I(sbi);
321 return &sit_i->sec_entries[GET_SEC_FROM_SEG(sbi, segno)];
322}
323
324static inline unsigned int get_valid_blocks(struct f2fs_sb_info *sbi,
325 unsigned int segno, bool use_section)
326{
327
328
329
330
331 if (use_section && sbi->segs_per_sec > 1)
332 return get_sec_entry(sbi, segno)->valid_blocks;
333 else
334 return get_seg_entry(sbi, segno)->valid_blocks;
335}
336
337static inline void seg_info_from_raw_sit(struct seg_entry *se,
338 struct f2fs_sit_entry *rs)
339{
340 se->valid_blocks = GET_SIT_VBLOCKS(rs);
341 se->ckpt_valid_blocks = GET_SIT_VBLOCKS(rs);
342 memcpy(se->cur_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE);
343 memcpy(se->ckpt_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE);
344#ifdef CONFIG_F2FS_CHECK_FS
345 memcpy(se->cur_valid_map_mir, rs->valid_map, SIT_VBLOCK_MAP_SIZE);
346#endif
347 se->type = GET_SIT_TYPE(rs);
348 se->mtime = le64_to_cpu(rs->mtime);
349}
350
351static inline void seg_info_to_raw_sit(struct seg_entry *se,
352 struct f2fs_sit_entry *rs)
353{
354 unsigned short raw_vblocks = (se->type << SIT_VBLOCKS_SHIFT) |
355 se->valid_blocks;
356 rs->vblocks = cpu_to_le16(raw_vblocks);
357 memcpy(rs->valid_map, se->cur_valid_map, SIT_VBLOCK_MAP_SIZE);
358 memcpy(se->ckpt_valid_map, rs->valid_map, SIT_VBLOCK_MAP_SIZE);
359 se->ckpt_valid_blocks = se->valid_blocks;
360 rs->mtime = cpu_to_le64(se->mtime);
361}
362
363static inline unsigned int find_next_inuse(struct free_segmap_info *free_i,
364 unsigned int max, unsigned int segno)
365{
366 unsigned int ret;
367 spin_lock(&free_i->segmap_lock);
368 ret = find_next_bit(free_i->free_segmap, max, segno);
369 spin_unlock(&free_i->segmap_lock);
370 return ret;
371}
372
373static inline void __set_free(struct f2fs_sb_info *sbi, unsigned int segno)
374{
375 struct free_segmap_info *free_i = FREE_I(sbi);
376 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
377 unsigned int start_segno = GET_SEG_FROM_SEC(sbi, secno);
378 unsigned int next;
379
380 spin_lock(&free_i->segmap_lock);
381 clear_bit(segno, free_i->free_segmap);
382 free_i->free_segments++;
383
384 next = find_next_bit(free_i->free_segmap,
385 start_segno + sbi->segs_per_sec, start_segno);
386 if (next >= start_segno + sbi->segs_per_sec) {
387 clear_bit(secno, free_i->free_secmap);
388 free_i->free_sections++;
389 }
390 spin_unlock(&free_i->segmap_lock);
391}
392
393static inline void __set_inuse(struct f2fs_sb_info *sbi,
394 unsigned int segno)
395{
396 struct free_segmap_info *free_i = FREE_I(sbi);
397 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
398
399 set_bit(segno, free_i->free_segmap);
400 free_i->free_segments--;
401 if (!test_and_set_bit(secno, free_i->free_secmap))
402 free_i->free_sections--;
403}
404
405static inline void __set_test_and_free(struct f2fs_sb_info *sbi,
406 unsigned int segno)
407{
408 struct free_segmap_info *free_i = FREE_I(sbi);
409 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
410 unsigned int start_segno = GET_SEG_FROM_SEC(sbi, secno);
411 unsigned int next;
412
413 spin_lock(&free_i->segmap_lock);
414 if (test_and_clear_bit(segno, free_i->free_segmap)) {
415 free_i->free_segments++;
416
417 next = find_next_bit(free_i->free_segmap,
418 start_segno + sbi->segs_per_sec, start_segno);
419 if (next >= start_segno + sbi->segs_per_sec) {
420 if (test_and_clear_bit(secno, free_i->free_secmap))
421 free_i->free_sections++;
422 }
423 }
424 spin_unlock(&free_i->segmap_lock);
425}
426
427static inline void __set_test_and_inuse(struct f2fs_sb_info *sbi,
428 unsigned int segno)
429{
430 struct free_segmap_info *free_i = FREE_I(sbi);
431 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
432
433 spin_lock(&free_i->segmap_lock);
434 if (!test_and_set_bit(segno, free_i->free_segmap)) {
435 free_i->free_segments--;
436 if (!test_and_set_bit(secno, free_i->free_secmap))
437 free_i->free_sections--;
438 }
439 spin_unlock(&free_i->segmap_lock);
440}
441
442static inline void get_sit_bitmap(struct f2fs_sb_info *sbi,
443 void *dst_addr)
444{
445 struct sit_info *sit_i = SIT_I(sbi);
446
447#ifdef CONFIG_F2FS_CHECK_FS
448 if (memcmp(sit_i->sit_bitmap, sit_i->sit_bitmap_mir,
449 sit_i->bitmap_size))
450 f2fs_bug_on(sbi, 1);
451#endif
452 memcpy(dst_addr, sit_i->sit_bitmap, sit_i->bitmap_size);
453}
454
455static inline block_t written_block_count(struct f2fs_sb_info *sbi)
456{
457 return SIT_I(sbi)->written_valid_blocks;
458}
459
460static inline unsigned int free_segments(struct f2fs_sb_info *sbi)
461{
462 return FREE_I(sbi)->free_segments;
463}
464
465static inline int reserved_segments(struct f2fs_sb_info *sbi)
466{
467 return SM_I(sbi)->reserved_segments;
468}
469
470static inline unsigned int free_sections(struct f2fs_sb_info *sbi)
471{
472 return FREE_I(sbi)->free_sections;
473}
474
475static inline unsigned int prefree_segments(struct f2fs_sb_info *sbi)
476{
477 return DIRTY_I(sbi)->nr_dirty[PRE];
478}
479
480static inline unsigned int dirty_segments(struct f2fs_sb_info *sbi)
481{
482 return DIRTY_I(sbi)->nr_dirty[DIRTY_HOT_DATA] +
483 DIRTY_I(sbi)->nr_dirty[DIRTY_WARM_DATA] +
484 DIRTY_I(sbi)->nr_dirty[DIRTY_COLD_DATA] +
485 DIRTY_I(sbi)->nr_dirty[DIRTY_HOT_NODE] +
486 DIRTY_I(sbi)->nr_dirty[DIRTY_WARM_NODE] +
487 DIRTY_I(sbi)->nr_dirty[DIRTY_COLD_NODE];
488}
489
490static inline int overprovision_segments(struct f2fs_sb_info *sbi)
491{
492 return SM_I(sbi)->ovp_segments;
493}
494
495static inline int reserved_sections(struct f2fs_sb_info *sbi)
496{
497 return GET_SEC_FROM_SEG(sbi, (unsigned int)reserved_segments(sbi));
498}
499
500static inline bool has_curseg_enough_space(struct f2fs_sb_info *sbi)
501{
502 unsigned int node_blocks = get_pages(sbi, F2FS_DIRTY_NODES) +
503 get_pages(sbi, F2FS_DIRTY_DENTS);
504 unsigned int dent_blocks = get_pages(sbi, F2FS_DIRTY_DENTS);
505 unsigned int segno, left_blocks;
506 int i;
507
508
509 for (i = CURSEG_HOT_NODE; i <= CURSEG_COLD_NODE; i++) {
510 segno = CURSEG_I(sbi, i)->segno;
511 left_blocks = sbi->blocks_per_seg -
512 get_seg_entry(sbi, segno)->ckpt_valid_blocks;
513
514 if (node_blocks > left_blocks)
515 return false;
516 }
517
518
519 segno = CURSEG_I(sbi, CURSEG_HOT_DATA)->segno;
520 left_blocks = sbi->blocks_per_seg -
521 get_seg_entry(sbi, segno)->ckpt_valid_blocks;
522 if (dent_blocks > left_blocks)
523 return false;
524 return true;
525}
526
527static inline bool has_not_enough_free_secs(struct f2fs_sb_info *sbi,
528 int freed, int needed)
529{
530 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
531 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
532 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
533
534 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
535 return false;
536
537 if (free_sections(sbi) + freed == reserved_sections(sbi) + needed &&
538 has_curseg_enough_space(sbi))
539 return false;
540 return (free_sections(sbi) + freed) <=
541 (node_secs + 2 * dent_secs + imeta_secs +
542 reserved_sections(sbi) + needed);
543}
544
545static inline bool excess_prefree_segs(struct f2fs_sb_info *sbi)
546{
547 return prefree_segments(sbi) > SM_I(sbi)->rec_prefree_segments;
548}
549
550static inline int utilization(struct f2fs_sb_info *sbi)
551{
552 return div_u64((u64)valid_user_blocks(sbi) * 100,
553 sbi->user_block_count);
554}
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570#define DEF_MIN_IPU_UTIL 70
571#define DEF_MIN_FSYNC_BLOCKS 8
572#define DEF_MIN_HOT_BLOCKS 16
573
574enum {
575 F2FS_IPU_FORCE,
576 F2FS_IPU_SSR,
577 F2FS_IPU_UTIL,
578 F2FS_IPU_SSR_UTIL,
579 F2FS_IPU_FSYNC,
580 F2FS_IPU_ASYNC,
581};
582
583static inline bool need_inplace_update_policy(struct inode *inode,
584 struct f2fs_io_info *fio)
585{
586 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
587 unsigned int policy = SM_I(sbi)->ipu_policy;
588
589 if (test_opt(sbi, LFS))
590 return false;
591
592
593 if (file_is_cold(inode))
594 return true;
595
596 if (policy & (0x1 << F2FS_IPU_FORCE))
597 return true;
598 if (policy & (0x1 << F2FS_IPU_SSR) && need_SSR(sbi))
599 return true;
600 if (policy & (0x1 << F2FS_IPU_UTIL) &&
601 utilization(sbi) > SM_I(sbi)->min_ipu_util)
602 return true;
603 if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && need_SSR(sbi) &&
604 utilization(sbi) > SM_I(sbi)->min_ipu_util)
605 return true;
606
607
608
609
610 if (policy & (0x1 << F2FS_IPU_ASYNC) &&
611 fio && fio->op == REQ_OP_WRITE &&
612 !(fio->op_flags & REQ_SYNC) &&
613 !f2fs_encrypted_inode(inode))
614 return true;
615
616
617 if (policy & (0x1 << F2FS_IPU_FSYNC) &&
618 is_inode_flag_set(inode, FI_NEED_IPU))
619 return true;
620
621 return false;
622}
623
624static inline unsigned int curseg_segno(struct f2fs_sb_info *sbi,
625 int type)
626{
627 struct curseg_info *curseg = CURSEG_I(sbi, type);
628 return curseg->segno;
629}
630
631static inline unsigned char curseg_alloc_type(struct f2fs_sb_info *sbi,
632 int type)
633{
634 struct curseg_info *curseg = CURSEG_I(sbi, type);
635 return curseg->alloc_type;
636}
637
638static inline unsigned short curseg_blkoff(struct f2fs_sb_info *sbi, int type)
639{
640 struct curseg_info *curseg = CURSEG_I(sbi, type);
641 return curseg->next_blkoff;
642}
643
644static inline void check_seg_range(struct f2fs_sb_info *sbi, unsigned int segno)
645{
646 f2fs_bug_on(sbi, segno > TOTAL_SEGS(sbi) - 1);
647}
648
649static inline void verify_block_addr(struct f2fs_sb_info *sbi, block_t blk_addr)
650{
651 BUG_ON(blk_addr < SEG0_BLKADDR(sbi)
652 || blk_addr >= MAX_BLKADDR(sbi));
653}
654
655
656
657
658static inline void check_block_count(struct f2fs_sb_info *sbi,
659 int segno, struct f2fs_sit_entry *raw_sit)
660{
661#ifdef CONFIG_F2FS_CHECK_FS
662 bool is_valid = test_bit_le(0, raw_sit->valid_map) ? true : false;
663 int valid_blocks = 0;
664 int cur_pos = 0, next_pos;
665
666
667 do {
668 if (is_valid) {
669 next_pos = find_next_zero_bit_le(&raw_sit->valid_map,
670 sbi->blocks_per_seg,
671 cur_pos);
672 valid_blocks += next_pos - cur_pos;
673 } else
674 next_pos = find_next_bit_le(&raw_sit->valid_map,
675 sbi->blocks_per_seg,
676 cur_pos);
677 cur_pos = next_pos;
678 is_valid = !is_valid;
679 } while (cur_pos < sbi->blocks_per_seg);
680 BUG_ON(GET_SIT_VBLOCKS(raw_sit) != valid_blocks);
681#endif
682
683 f2fs_bug_on(sbi, GET_SIT_VBLOCKS(raw_sit) > sbi->blocks_per_seg
684 || segno > TOTAL_SEGS(sbi) - 1);
685}
686
687static inline pgoff_t current_sit_addr(struct f2fs_sb_info *sbi,
688 unsigned int start)
689{
690 struct sit_info *sit_i = SIT_I(sbi);
691 unsigned int offset = SIT_BLOCK_OFFSET(start);
692 block_t blk_addr = sit_i->sit_base_addr + offset;
693
694 check_seg_range(sbi, start);
695
696#ifdef CONFIG_F2FS_CHECK_FS
697 if (f2fs_test_bit(offset, sit_i->sit_bitmap) !=
698 f2fs_test_bit(offset, sit_i->sit_bitmap_mir))
699 f2fs_bug_on(sbi, 1);
700#endif
701
702
703 if (f2fs_test_bit(offset, sit_i->sit_bitmap))
704 blk_addr += sit_i->sit_blocks;
705
706 return blk_addr;
707}
708
709static inline pgoff_t next_sit_addr(struct f2fs_sb_info *sbi,
710 pgoff_t block_addr)
711{
712 struct sit_info *sit_i = SIT_I(sbi);
713 block_addr -= sit_i->sit_base_addr;
714 if (block_addr < sit_i->sit_blocks)
715 block_addr += sit_i->sit_blocks;
716 else
717 block_addr -= sit_i->sit_blocks;
718
719 return block_addr + sit_i->sit_base_addr;
720}
721
722static inline void set_to_next_sit(struct sit_info *sit_i, unsigned int start)
723{
724 unsigned int block_off = SIT_BLOCK_OFFSET(start);
725
726 f2fs_change_bit(block_off, sit_i->sit_bitmap);
727#ifdef CONFIG_F2FS_CHECK_FS
728 f2fs_change_bit(block_off, sit_i->sit_bitmap_mir);
729#endif
730}
731
732static inline unsigned long long get_mtime(struct f2fs_sb_info *sbi)
733{
734 struct sit_info *sit_i = SIT_I(sbi);
735 time64_t now = ktime_get_real_seconds();
736
737 return sit_i->elapsed_time + now - sit_i->mounted_time;
738}
739
740static inline void set_summary(struct f2fs_summary *sum, nid_t nid,
741 unsigned int ofs_in_node, unsigned char version)
742{
743 sum->nid = cpu_to_le32(nid);
744 sum->ofs_in_node = cpu_to_le16(ofs_in_node);
745 sum->version = version;
746}
747
748static inline block_t start_sum_block(struct f2fs_sb_info *sbi)
749{
750 return __start_cp_addr(sbi) +
751 le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_start_sum);
752}
753
754static inline block_t sum_blk_addr(struct f2fs_sb_info *sbi, int base, int type)
755{
756 return __start_cp_addr(sbi) +
757 le32_to_cpu(F2FS_CKPT(sbi)->cp_pack_total_block_count)
758 - (base + 1) + type;
759}
760
761static inline bool no_fggc_candidate(struct f2fs_sb_info *sbi,
762 unsigned int secno)
763{
764 if (get_valid_blocks(sbi, GET_SEG_FROM_SEC(sbi, secno), true) >
765 sbi->fggc_threshold)
766 return true;
767 return false;
768}
769
770static inline bool sec_usage_check(struct f2fs_sb_info *sbi, unsigned int secno)
771{
772 if (IS_CURSEC(sbi, secno) || (sbi->cur_victim_sec == secno))
773 return true;
774 return false;
775}
776
777
778
779
780
781
782
783
784static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type)
785{
786 if (sbi->sb->s_bdi->wb.dirty_exceeded)
787 return 0;
788
789 if (type == DATA)
790 return sbi->blocks_per_seg;
791 else if (type == NODE)
792 return 8 * sbi->blocks_per_seg;
793 else if (type == META)
794 return 8 * BIO_MAX_PAGES;
795 else
796 return 0;
797}
798
799
800
801
802static inline long nr_pages_to_write(struct f2fs_sb_info *sbi, int type,
803 struct writeback_control *wbc)
804{
805 long nr_to_write, desired;
806
807 if (wbc->sync_mode != WB_SYNC_NONE)
808 return 0;
809
810 nr_to_write = wbc->nr_to_write;
811 desired = BIO_MAX_PAGES;
812 if (type == NODE)
813 desired <<= 1;
814
815 wbc->nr_to_write = desired;
816 return desired - nr_to_write;
817}
818
819static inline void wake_up_discard_thread(struct f2fs_sb_info *sbi, bool force)
820{
821 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
822 bool wakeup = false;
823 int i;
824
825 if (force)
826 goto wake_up;
827
828 mutex_lock(&dcc->cmd_lock);
829 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
830 if (i + 1 < dcc->discard_granularity)
831 break;
832 if (!list_empty(&dcc->pend_list[i])) {
833 wakeup = true;
834 break;
835 }
836 }
837 mutex_unlock(&dcc->cmd_lock);
838 if (!wakeup)
839 return;
840wake_up:
841 dcc->discard_wake = 1;
842 wake_up_interruptible_all(&dcc->discard_wait_queue);
843}
844