1
2
3
4
5
6
7
8#include <linux/fs.h>
9#include <linux/f2fs_fs.h>
10#include <linux/bio.h>
11#include <linux/blkdev.h>
12#include <linux/sched/mm.h>
13#include <linux/prefetch.h>
14#include <linux/kthread.h>
15#include <linux/swap.h>
16#include <linux/timer.h>
17#include <linux/freezer.h>
18#include <linux/sched/signal.h>
19#include <linux/random.h>
20
21#include "f2fs.h"
22#include "segment.h"
23#include "node.h"
24#include "gc.h"
25#include "iostat.h"
26#include <trace/events/f2fs.h>
27
28#define __reverse_ffz(x) __reverse_ffs(~(x))
29
30static struct kmem_cache *discard_entry_slab;
31static struct kmem_cache *discard_cmd_slab;
32static struct kmem_cache *sit_entry_set_slab;
33static struct kmem_cache *inmem_entry_slab;
34
35static unsigned long __reverse_ulong(unsigned char *str)
36{
37 unsigned long tmp = 0;
38 int shift = 24, idx = 0;
39
40#if BITS_PER_LONG == 64
41 shift = 56;
42#endif
43 while (shift >= 0) {
44 tmp |= (unsigned long)str[idx++] << shift;
45 shift -= BITS_PER_BYTE;
46 }
47 return tmp;
48}
49
50
51
52
53
54static inline unsigned long __reverse_ffs(unsigned long word)
55{
56 int num = 0;
57
58#if BITS_PER_LONG == 64
59 if ((word & 0xffffffff00000000UL) == 0)
60 num += 32;
61 else
62 word >>= 32;
63#endif
64 if ((word & 0xffff0000) == 0)
65 num += 16;
66 else
67 word >>= 16;
68
69 if ((word & 0xff00) == 0)
70 num += 8;
71 else
72 word >>= 8;
73
74 if ((word & 0xf0) == 0)
75 num += 4;
76 else
77 word >>= 4;
78
79 if ((word & 0xc) == 0)
80 num += 2;
81 else
82 word >>= 2;
83
84 if ((word & 0x2) == 0)
85 num += 1;
86 return num;
87}
88
89
90
91
92
93
94
95
96
97
98static unsigned long __find_rev_next_bit(const unsigned long *addr,
99 unsigned long size, unsigned long offset)
100{
101 const unsigned long *p = addr + BIT_WORD(offset);
102 unsigned long result = size;
103 unsigned long tmp;
104
105 if (offset >= size)
106 return size;
107
108 size -= (offset & ~(BITS_PER_LONG - 1));
109 offset %= BITS_PER_LONG;
110
111 while (1) {
112 if (*p == 0)
113 goto pass;
114
115 tmp = __reverse_ulong((unsigned char *)p);
116
117 tmp &= ~0UL >> offset;
118 if (size < BITS_PER_LONG)
119 tmp &= (~0UL << (BITS_PER_LONG - size));
120 if (tmp)
121 goto found;
122pass:
123 if (size <= BITS_PER_LONG)
124 break;
125 size -= BITS_PER_LONG;
126 offset = 0;
127 p++;
128 }
129 return result;
130found:
131 return result - size + __reverse_ffs(tmp);
132}
133
134static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
135 unsigned long size, unsigned long offset)
136{
137 const unsigned long *p = addr + BIT_WORD(offset);
138 unsigned long result = size;
139 unsigned long tmp;
140
141 if (offset >= size)
142 return size;
143
144 size -= (offset & ~(BITS_PER_LONG - 1));
145 offset %= BITS_PER_LONG;
146
147 while (1) {
148 if (*p == ~0UL)
149 goto pass;
150
151 tmp = __reverse_ulong((unsigned char *)p);
152
153 if (offset)
154 tmp |= ~0UL << (BITS_PER_LONG - offset);
155 if (size < BITS_PER_LONG)
156 tmp |= ~0UL >> size;
157 if (tmp != ~0UL)
158 goto found;
159pass:
160 if (size <= BITS_PER_LONG)
161 break;
162 size -= BITS_PER_LONG;
163 offset = 0;
164 p++;
165 }
166 return result;
167found:
168 return result - size + __reverse_ffz(tmp);
169}
170
171bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
172{
173 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
174 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
175 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
176
177 if (f2fs_lfs_mode(sbi))
178 return false;
179 if (sbi->gc_mode == GC_URGENT_HIGH)
180 return true;
181 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
182 return true;
183
184 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
185 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
186}
187
188void f2fs_register_inmem_page(struct inode *inode, struct page *page)
189{
190 struct inmem_pages *new;
191
192 set_page_private_atomic(page);
193
194 new = f2fs_kmem_cache_alloc(inmem_entry_slab,
195 GFP_NOFS, true, NULL);
196
197
198 new->page = page;
199 INIT_LIST_HEAD(&new->list);
200
201
202 get_page(page);
203 mutex_lock(&F2FS_I(inode)->inmem_lock);
204 list_add_tail(&new->list, &F2FS_I(inode)->inmem_pages);
205 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
206 mutex_unlock(&F2FS_I(inode)->inmem_lock);
207
208 trace_f2fs_register_inmem_page(page, INMEM);
209}
210
211static int __revoke_inmem_pages(struct inode *inode,
212 struct list_head *head, bool drop, bool recover,
213 bool trylock)
214{
215 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
216 struct inmem_pages *cur, *tmp;
217 int err = 0;
218
219 list_for_each_entry_safe(cur, tmp, head, list) {
220 struct page *page = cur->page;
221
222 if (drop)
223 trace_f2fs_commit_inmem_page(page, INMEM_DROP);
224
225 if (trylock) {
226
227
228
229
230 if (!trylock_page(page))
231 continue;
232 } else {
233 lock_page(page);
234 }
235
236 f2fs_wait_on_page_writeback(page, DATA, true, true);
237
238 if (recover) {
239 struct dnode_of_data dn;
240 struct node_info ni;
241
242 trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
243retry:
244 set_new_dnode(&dn, inode, NULL, NULL, 0);
245 err = f2fs_get_dnode_of_data(&dn, page->index,
246 LOOKUP_NODE);
247 if (err) {
248 if (err == -ENOMEM) {
249 memalloc_retry_wait(GFP_NOFS);
250 goto retry;
251 }
252 err = -EAGAIN;
253 goto next;
254 }
255
256 err = f2fs_get_node_info(sbi, dn.nid, &ni, false);
257 if (err) {
258 f2fs_put_dnode(&dn);
259 return err;
260 }
261
262 if (cur->old_addr == NEW_ADDR) {
263 f2fs_invalidate_blocks(sbi, dn.data_blkaddr);
264 f2fs_update_data_blkaddr(&dn, NEW_ADDR);
265 } else
266 f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
267 cur->old_addr, ni.version, true, true);
268 f2fs_put_dnode(&dn);
269 }
270next:
271
272 if (drop || recover) {
273 ClearPageUptodate(page);
274 clear_page_private_gcing(page);
275 }
276 detach_page_private(page);
277 set_page_private(page, 0);
278 f2fs_put_page(page, 1);
279
280 list_del(&cur->list);
281 kmem_cache_free(inmem_entry_slab, cur);
282 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
283 }
284 return err;
285}
286
287void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)
288{
289 struct list_head *head = &sbi->inode_list[ATOMIC_FILE];
290 struct inode *inode;
291 struct f2fs_inode_info *fi;
292 unsigned int count = sbi->atomic_files;
293 unsigned int looped = 0;
294next:
295 spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
296 if (list_empty(head)) {
297 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
298 return;
299 }
300 fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist);
301 inode = igrab(&fi->vfs_inode);
302 if (inode)
303 list_move_tail(&fi->inmem_ilist, head);
304 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
305
306 if (inode) {
307 if (gc_failure) {
308 if (!fi->i_gc_failures[GC_FAILURE_ATOMIC])
309 goto skip;
310 }
311 set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST);
312 f2fs_drop_inmem_pages(inode);
313skip:
314 iput(inode);
315 }
316 congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
317 cond_resched();
318 if (gc_failure) {
319 if (++looped >= count)
320 return;
321 }
322 goto next;
323}
324
325void f2fs_drop_inmem_pages(struct inode *inode)
326{
327 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
328 struct f2fs_inode_info *fi = F2FS_I(inode);
329
330 do {
331 mutex_lock(&fi->inmem_lock);
332 if (list_empty(&fi->inmem_pages)) {
333 fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0;
334
335 spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
336 if (!list_empty(&fi->inmem_ilist))
337 list_del_init(&fi->inmem_ilist);
338 if (f2fs_is_atomic_file(inode)) {
339 clear_inode_flag(inode, FI_ATOMIC_FILE);
340 sbi->atomic_files--;
341 }
342 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
343
344 mutex_unlock(&fi->inmem_lock);
345 break;
346 }
347 __revoke_inmem_pages(inode, &fi->inmem_pages,
348 true, false, true);
349 mutex_unlock(&fi->inmem_lock);
350 } while (1);
351}
352
353void f2fs_drop_inmem_page(struct inode *inode, struct page *page)
354{
355 struct f2fs_inode_info *fi = F2FS_I(inode);
356 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
357 struct list_head *head = &fi->inmem_pages;
358 struct inmem_pages *cur = NULL;
359
360 f2fs_bug_on(sbi, !page_private_atomic(page));
361
362 mutex_lock(&fi->inmem_lock);
363 list_for_each_entry(cur, head, list) {
364 if (cur->page == page)
365 break;
366 }
367
368 f2fs_bug_on(sbi, list_empty(head) || cur->page != page);
369 list_del(&cur->list);
370 mutex_unlock(&fi->inmem_lock);
371
372 dec_page_count(sbi, F2FS_INMEM_PAGES);
373 kmem_cache_free(inmem_entry_slab, cur);
374
375 ClearPageUptodate(page);
376 clear_page_private_atomic(page);
377 f2fs_put_page(page, 0);
378
379 detach_page_private(page);
380 set_page_private(page, 0);
381
382 trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE);
383}
384
385static int __f2fs_commit_inmem_pages(struct inode *inode)
386{
387 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
388 struct f2fs_inode_info *fi = F2FS_I(inode);
389 struct inmem_pages *cur, *tmp;
390 struct f2fs_io_info fio = {
391 .sbi = sbi,
392 .ino = inode->i_ino,
393 .type = DATA,
394 .op = REQ_OP_WRITE,
395 .op_flags = REQ_SYNC | REQ_PRIO,
396 .io_type = FS_DATA_IO,
397 };
398 struct list_head revoke_list;
399 bool submit_bio = false;
400 int err = 0;
401
402 INIT_LIST_HEAD(&revoke_list);
403
404 list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
405 struct page *page = cur->page;
406
407 lock_page(page);
408 if (page->mapping == inode->i_mapping) {
409 trace_f2fs_commit_inmem_page(page, INMEM);
410
411 f2fs_wait_on_page_writeback(page, DATA, true, true);
412
413 set_page_dirty(page);
414 if (clear_page_dirty_for_io(page)) {
415 inode_dec_dirty_pages(inode);
416 f2fs_remove_dirty_inode(inode);
417 }
418retry:
419 fio.page = page;
420 fio.old_blkaddr = NULL_ADDR;
421 fio.encrypted_page = NULL;
422 fio.need_lock = LOCK_DONE;
423 err = f2fs_do_write_data_page(&fio);
424 if (err) {
425 if (err == -ENOMEM) {
426 memalloc_retry_wait(GFP_NOFS);
427 goto retry;
428 }
429 unlock_page(page);
430 break;
431 }
432
433 cur->old_addr = fio.old_blkaddr;
434 submit_bio = true;
435 }
436 unlock_page(page);
437 list_move_tail(&cur->list, &revoke_list);
438 }
439
440 if (submit_bio)
441 f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA);
442
443 if (err) {
444
445
446
447
448
449
450
451
452 err = __revoke_inmem_pages(inode, &revoke_list,
453 false, true, false);
454
455
456 __revoke_inmem_pages(inode, &fi->inmem_pages,
457 true, false, false);
458 } else {
459 __revoke_inmem_pages(inode, &revoke_list,
460 false, false, false);
461 }
462
463 return err;
464}
465
466int f2fs_commit_inmem_pages(struct inode *inode)
467{
468 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
469 struct f2fs_inode_info *fi = F2FS_I(inode);
470 int err;
471
472 f2fs_balance_fs(sbi, true);
473
474 down_write(&fi->i_gc_rwsem[WRITE]);
475
476 f2fs_lock_op(sbi);
477 set_inode_flag(inode, FI_ATOMIC_COMMIT);
478
479 mutex_lock(&fi->inmem_lock);
480 err = __f2fs_commit_inmem_pages(inode);
481 mutex_unlock(&fi->inmem_lock);
482
483 clear_inode_flag(inode, FI_ATOMIC_COMMIT);
484
485 f2fs_unlock_op(sbi);
486 up_write(&fi->i_gc_rwsem[WRITE]);
487
488 return err;
489}
490
491
492
493
494
495void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
496{
497 if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
498 f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
499 f2fs_stop_checkpoint(sbi, false);
500 }
501
502
503 if (need && excess_cached_nats(sbi))
504 f2fs_balance_fs_bg(sbi, false);
505
506 if (!f2fs_is_checkpoint_ready(sbi))
507 return;
508
509
510
511
512
513 if (has_not_enough_free_secs(sbi, 0, 0)) {
514 if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
515 sbi->gc_thread->f2fs_gc_task) {
516 DEFINE_WAIT(wait);
517
518 prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
519 TASK_UNINTERRUPTIBLE);
520 wake_up(&sbi->gc_thread->gc_wait_queue_head);
521 io_schedule();
522 finish_wait(&sbi->gc_thread->fggc_wq, &wait);
523 } else {
524 down_write(&sbi->gc_lock);
525 f2fs_gc(sbi, false, false, false, NULL_SEGNO);
526 }
527 }
528}
529
530static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi)
531{
532 int factor = rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2;
533 unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS);
534 unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA);
535 unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES);
536 unsigned int meta = get_pages(sbi, F2FS_DIRTY_META);
537 unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA);
538 unsigned int threshold = sbi->blocks_per_seg * factor *
539 DEFAULT_DIRTY_THRESHOLD;
540 unsigned int global_threshold = threshold * 3 / 2;
541
542 if (dents >= threshold || qdata >= threshold ||
543 nodes >= threshold || meta >= threshold ||
544 imeta >= threshold)
545 return true;
546 return dents + qdata + nodes + meta + imeta > global_threshold;
547}
548
549void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
550{
551 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
552 return;
553
554
555 if (!f2fs_available_free_memory(sbi, EXTENT_CACHE))
556 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
557
558
559 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
560 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
561
562 if (!f2fs_available_free_memory(sbi, FREE_NIDS))
563 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
564 else
565 f2fs_build_free_nids(sbi, false, false);
566
567 if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) ||
568 excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi))
569 goto do_sync;
570
571
572 if (is_inflight_io(sbi, REQ_TIME) ||
573 (!f2fs_time_over(sbi, REQ_TIME) && rwsem_is_locked(&sbi->cp_rwsem)))
574 return;
575
576
577 if (f2fs_time_over(sbi, CP_TIME))
578 goto do_sync;
579
580
581 if (f2fs_available_free_memory(sbi, NAT_ENTRIES) &&
582 f2fs_available_free_memory(sbi, INO_ENTRIES))
583 return;
584
585do_sync:
586 if (test_opt(sbi, DATA_FLUSH) && from_bg) {
587 struct blk_plug plug;
588
589 mutex_lock(&sbi->flush_lock);
590
591 blk_start_plug(&plug);
592 f2fs_sync_dirty_inodes(sbi, FILE_INODE);
593 blk_finish_plug(&plug);
594
595 mutex_unlock(&sbi->flush_lock);
596 }
597 f2fs_sync_fs(sbi->sb, true);
598 stat_inc_bg_cp_count(sbi->stat_info);
599}
600
601static int __submit_flush_wait(struct f2fs_sb_info *sbi,
602 struct block_device *bdev)
603{
604 int ret = blkdev_issue_flush(bdev);
605
606 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
607 test_opt(sbi, FLUSH_MERGE), ret);
608 return ret;
609}
610
611static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
612{
613 int ret = 0;
614 int i;
615
616 if (!f2fs_is_multi_device(sbi))
617 return __submit_flush_wait(sbi, sbi->sb->s_bdev);
618
619 for (i = 0; i < sbi->s_ndevs; i++) {
620 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
621 continue;
622 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
623 if (ret)
624 break;
625 }
626 return ret;
627}
628
629static int issue_flush_thread(void *data)
630{
631 struct f2fs_sb_info *sbi = data;
632 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
633 wait_queue_head_t *q = &fcc->flush_wait_queue;
634repeat:
635 if (kthread_should_stop())
636 return 0;
637
638 if (!llist_empty(&fcc->issue_list)) {
639 struct flush_cmd *cmd, *next;
640 int ret;
641
642 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
643 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
644
645 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
646
647 ret = submit_flush_wait(sbi, cmd->ino);
648 atomic_inc(&fcc->issued_flush);
649
650 llist_for_each_entry_safe(cmd, next,
651 fcc->dispatch_list, llnode) {
652 cmd->ret = ret;
653 complete(&cmd->wait);
654 }
655 fcc->dispatch_list = NULL;
656 }
657
658 wait_event_interruptible(*q,
659 kthread_should_stop() || !llist_empty(&fcc->issue_list));
660 goto repeat;
661}
662
663int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
664{
665 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
666 struct flush_cmd cmd;
667 int ret;
668
669 if (test_opt(sbi, NOBARRIER))
670 return 0;
671
672 if (!test_opt(sbi, FLUSH_MERGE)) {
673 atomic_inc(&fcc->queued_flush);
674 ret = submit_flush_wait(sbi, ino);
675 atomic_dec(&fcc->queued_flush);
676 atomic_inc(&fcc->issued_flush);
677 return ret;
678 }
679
680 if (atomic_inc_return(&fcc->queued_flush) == 1 ||
681 f2fs_is_multi_device(sbi)) {
682 ret = submit_flush_wait(sbi, ino);
683 atomic_dec(&fcc->queued_flush);
684
685 atomic_inc(&fcc->issued_flush);
686 return ret;
687 }
688
689 cmd.ino = ino;
690 init_completion(&cmd.wait);
691
692 llist_add(&cmd.llnode, &fcc->issue_list);
693
694
695
696
697
698
699 smp_mb();
700
701 if (waitqueue_active(&fcc->flush_wait_queue))
702 wake_up(&fcc->flush_wait_queue);
703
704 if (fcc->f2fs_issue_flush) {
705 wait_for_completion(&cmd.wait);
706 atomic_dec(&fcc->queued_flush);
707 } else {
708 struct llist_node *list;
709
710 list = llist_del_all(&fcc->issue_list);
711 if (!list) {
712 wait_for_completion(&cmd.wait);
713 atomic_dec(&fcc->queued_flush);
714 } else {
715 struct flush_cmd *tmp, *next;
716
717 ret = submit_flush_wait(sbi, ino);
718
719 llist_for_each_entry_safe(tmp, next, list, llnode) {
720 if (tmp == &cmd) {
721 cmd.ret = ret;
722 atomic_dec(&fcc->queued_flush);
723 continue;
724 }
725 tmp->ret = ret;
726 complete(&tmp->wait);
727 }
728 }
729 }
730
731 return cmd.ret;
732}
733
734int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
735{
736 dev_t dev = sbi->sb->s_bdev->bd_dev;
737 struct flush_cmd_control *fcc;
738 int err = 0;
739
740 if (SM_I(sbi)->fcc_info) {
741 fcc = SM_I(sbi)->fcc_info;
742 if (fcc->f2fs_issue_flush)
743 return err;
744 goto init_thread;
745 }
746
747 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
748 if (!fcc)
749 return -ENOMEM;
750 atomic_set(&fcc->issued_flush, 0);
751 atomic_set(&fcc->queued_flush, 0);
752 init_waitqueue_head(&fcc->flush_wait_queue);
753 init_llist_head(&fcc->issue_list);
754 SM_I(sbi)->fcc_info = fcc;
755 if (!test_opt(sbi, FLUSH_MERGE))
756 return err;
757
758init_thread:
759 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
760 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
761 if (IS_ERR(fcc->f2fs_issue_flush)) {
762 err = PTR_ERR(fcc->f2fs_issue_flush);
763 kfree(fcc);
764 SM_I(sbi)->fcc_info = NULL;
765 return err;
766 }
767
768 return err;
769}
770
771void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
772{
773 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
774
775 if (fcc && fcc->f2fs_issue_flush) {
776 struct task_struct *flush_thread = fcc->f2fs_issue_flush;
777
778 fcc->f2fs_issue_flush = NULL;
779 kthread_stop(flush_thread);
780 }
781 if (free) {
782 kfree(fcc);
783 SM_I(sbi)->fcc_info = NULL;
784 }
785}
786
787int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
788{
789 int ret = 0, i;
790
791 if (!f2fs_is_multi_device(sbi))
792 return 0;
793
794 if (test_opt(sbi, NOBARRIER))
795 return 0;
796
797 for (i = 1; i < sbi->s_ndevs; i++) {
798 int count = DEFAULT_RETRY_IO_COUNT;
799
800 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
801 continue;
802
803 do {
804 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
805 if (ret)
806 congestion_wait(BLK_RW_ASYNC,
807 DEFAULT_IO_TIMEOUT);
808 } while (ret && --count);
809
810 if (ret) {
811 f2fs_stop_checkpoint(sbi, false);
812 break;
813 }
814
815 spin_lock(&sbi->dev_lock);
816 f2fs_clear_bit(i, (char *)&sbi->dirty_device);
817 spin_unlock(&sbi->dev_lock);
818 }
819
820 return ret;
821}
822
823static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
824 enum dirty_type dirty_type)
825{
826 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
827
828
829 if (IS_CURSEG(sbi, segno))
830 return;
831
832 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
833 dirty_i->nr_dirty[dirty_type]++;
834
835 if (dirty_type == DIRTY) {
836 struct seg_entry *sentry = get_seg_entry(sbi, segno);
837 enum dirty_type t = sentry->type;
838
839 if (unlikely(t >= DIRTY)) {
840 f2fs_bug_on(sbi, 1);
841 return;
842 }
843 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
844 dirty_i->nr_dirty[t]++;
845
846 if (__is_large_section(sbi)) {
847 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
848 block_t valid_blocks =
849 get_valid_blocks(sbi, segno, true);
850
851 f2fs_bug_on(sbi, unlikely(!valid_blocks ||
852 valid_blocks == BLKS_PER_SEC(sbi)));
853
854 if (!IS_CURSEC(sbi, secno))
855 set_bit(secno, dirty_i->dirty_secmap);
856 }
857 }
858}
859
860static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
861 enum dirty_type dirty_type)
862{
863 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
864 block_t valid_blocks;
865
866 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
867 dirty_i->nr_dirty[dirty_type]--;
868
869 if (dirty_type == DIRTY) {
870 struct seg_entry *sentry = get_seg_entry(sbi, segno);
871 enum dirty_type t = sentry->type;
872
873 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
874 dirty_i->nr_dirty[t]--;
875
876 valid_blocks = get_valid_blocks(sbi, segno, true);
877 if (valid_blocks == 0) {
878 clear_bit(GET_SEC_FROM_SEG(sbi, segno),
879 dirty_i->victim_secmap);
880#ifdef CONFIG_F2FS_CHECK_FS
881 clear_bit(segno, SIT_I(sbi)->invalid_segmap);
882#endif
883 }
884 if (__is_large_section(sbi)) {
885 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
886
887 if (!valid_blocks ||
888 valid_blocks == BLKS_PER_SEC(sbi)) {
889 clear_bit(secno, dirty_i->dirty_secmap);
890 return;
891 }
892
893 if (!IS_CURSEC(sbi, secno))
894 set_bit(secno, dirty_i->dirty_secmap);
895 }
896 }
897}
898
899
900
901
902
903
904static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
905{
906 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
907 unsigned short valid_blocks, ckpt_valid_blocks;
908 unsigned int usable_blocks;
909
910 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
911 return;
912
913 usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
914 mutex_lock(&dirty_i->seglist_lock);
915
916 valid_blocks = get_valid_blocks(sbi, segno, false);
917 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
918
919 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
920 ckpt_valid_blocks == usable_blocks)) {
921 __locate_dirty_segment(sbi, segno, PRE);
922 __remove_dirty_segment(sbi, segno, DIRTY);
923 } else if (valid_blocks < usable_blocks) {
924 __locate_dirty_segment(sbi, segno, DIRTY);
925 } else {
926
927 __remove_dirty_segment(sbi, segno, DIRTY);
928 }
929
930 mutex_unlock(&dirty_i->seglist_lock);
931}
932
933
934void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
935{
936 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
937 unsigned int segno;
938
939 mutex_lock(&dirty_i->seglist_lock);
940 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
941 if (get_valid_blocks(sbi, segno, false))
942 continue;
943 if (IS_CURSEG(sbi, segno))
944 continue;
945 __locate_dirty_segment(sbi, segno, PRE);
946 __remove_dirty_segment(sbi, segno, DIRTY);
947 }
948 mutex_unlock(&dirty_i->seglist_lock);
949}
950
951block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
952{
953 int ovp_hole_segs =
954 (overprovision_segments(sbi) - reserved_segments(sbi));
955 block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;
956 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
957 block_t holes[2] = {0, 0};
958 block_t unusable;
959 struct seg_entry *se;
960 unsigned int segno;
961
962 mutex_lock(&dirty_i->seglist_lock);
963 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
964 se = get_seg_entry(sbi, segno);
965 if (IS_NODESEG(se->type))
966 holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
967 se->valid_blocks;
968 else
969 holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
970 se->valid_blocks;
971 }
972 mutex_unlock(&dirty_i->seglist_lock);
973
974 unusable = holes[DATA] > holes[NODE] ? holes[DATA] : holes[NODE];
975 if (unusable > ovp_holes)
976 return unusable - ovp_holes;
977 return 0;
978}
979
980int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
981{
982 int ovp_hole_segs =
983 (overprovision_segments(sbi) - reserved_segments(sbi));
984 if (unusable > F2FS_OPTION(sbi).unusable_cap)
985 return -EAGAIN;
986 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
987 dirty_segments(sbi) > ovp_hole_segs)
988 return -EAGAIN;
989 return 0;
990}
991
992
993static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
994{
995 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
996 unsigned int segno = 0;
997
998 mutex_lock(&dirty_i->seglist_lock);
999 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
1000 if (get_valid_blocks(sbi, segno, false))
1001 continue;
1002 if (get_ckpt_valid_blocks(sbi, segno, false))
1003 continue;
1004 mutex_unlock(&dirty_i->seglist_lock);
1005 return segno;
1006 }
1007 mutex_unlock(&dirty_i->seglist_lock);
1008 return NULL_SEGNO;
1009}
1010
1011static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
1012 struct block_device *bdev, block_t lstart,
1013 block_t start, block_t len)
1014{
1015 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1016 struct list_head *pend_list;
1017 struct discard_cmd *dc;
1018
1019 f2fs_bug_on(sbi, !len);
1020
1021 pend_list = &dcc->pend_list[plist_idx(len)];
1022
1023 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL);
1024 INIT_LIST_HEAD(&dc->list);
1025 dc->bdev = bdev;
1026 dc->lstart = lstart;
1027 dc->start = start;
1028 dc->len = len;
1029 dc->ref = 0;
1030 dc->state = D_PREP;
1031 dc->queued = 0;
1032 dc->error = 0;
1033 init_completion(&dc->wait);
1034 list_add_tail(&dc->list, pend_list);
1035 spin_lock_init(&dc->lock);
1036 dc->bio_ref = 0;
1037 atomic_inc(&dcc->discard_cmd_cnt);
1038 dcc->undiscard_blks += len;
1039
1040 return dc;
1041}
1042
1043static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,
1044 struct block_device *bdev, block_t lstart,
1045 block_t start, block_t len,
1046 struct rb_node *parent, struct rb_node **p,
1047 bool leftmost)
1048{
1049 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1050 struct discard_cmd *dc;
1051
1052 dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
1053
1054 rb_link_node(&dc->rb_node, parent, p);
1055 rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
1056
1057 return dc;
1058}
1059
1060static void __detach_discard_cmd(struct discard_cmd_control *dcc,
1061 struct discard_cmd *dc)
1062{
1063 if (dc->state == D_DONE)
1064 atomic_sub(dc->queued, &dcc->queued_discard);
1065
1066 list_del(&dc->list);
1067 rb_erase_cached(&dc->rb_node, &dcc->root);
1068 dcc->undiscard_blks -= dc->len;
1069
1070 kmem_cache_free(discard_cmd_slab, dc);
1071
1072 atomic_dec(&dcc->discard_cmd_cnt);
1073}
1074
1075static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
1076 struct discard_cmd *dc)
1077{
1078 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1079 unsigned long flags;
1080
1081 trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len);
1082
1083 spin_lock_irqsave(&dc->lock, flags);
1084 if (dc->bio_ref) {
1085 spin_unlock_irqrestore(&dc->lock, flags);
1086 return;
1087 }
1088 spin_unlock_irqrestore(&dc->lock, flags);
1089
1090 f2fs_bug_on(sbi, dc->ref);
1091
1092 if (dc->error == -EOPNOTSUPP)
1093 dc->error = 0;
1094
1095 if (dc->error)
1096 printk_ratelimited(
1097 "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d",
1098 KERN_INFO, sbi->sb->s_id,
1099 dc->lstart, dc->start, dc->len, dc->error);
1100 __detach_discard_cmd(dcc, dc);
1101}
1102
1103static void f2fs_submit_discard_endio(struct bio *bio)
1104{
1105 struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
1106 unsigned long flags;
1107
1108 spin_lock_irqsave(&dc->lock, flags);
1109 if (!dc->error)
1110 dc->error = blk_status_to_errno(bio->bi_status);
1111 dc->bio_ref--;
1112 if (!dc->bio_ref && dc->state == D_SUBMIT) {
1113 dc->state = D_DONE;
1114 complete_all(&dc->wait);
1115 }
1116 spin_unlock_irqrestore(&dc->lock, flags);
1117 bio_put(bio);
1118}
1119
1120static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
1121 block_t start, block_t end)
1122{
1123#ifdef CONFIG_F2FS_CHECK_FS
1124 struct seg_entry *sentry;
1125 unsigned int segno;
1126 block_t blk = start;
1127 unsigned long offset, size, max_blocks = sbi->blocks_per_seg;
1128 unsigned long *map;
1129
1130 while (blk < end) {
1131 segno = GET_SEGNO(sbi, blk);
1132 sentry = get_seg_entry(sbi, segno);
1133 offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
1134
1135 if (end < START_BLOCK(sbi, segno + 1))
1136 size = GET_BLKOFF_FROM_SEG0(sbi, end);
1137 else
1138 size = max_blocks;
1139 map = (unsigned long *)(sentry->cur_valid_map);
1140 offset = __find_rev_next_bit(map, size, offset);
1141 f2fs_bug_on(sbi, offset != size);
1142 blk = START_BLOCK(sbi, segno + 1);
1143 }
1144#endif
1145}
1146
1147static void __init_discard_policy(struct f2fs_sb_info *sbi,
1148 struct discard_policy *dpolicy,
1149 int discard_type, unsigned int granularity)
1150{
1151 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1152
1153
1154 dpolicy->type = discard_type;
1155 dpolicy->sync = true;
1156 dpolicy->ordered = false;
1157 dpolicy->granularity = granularity;
1158
1159 dpolicy->max_requests = DEF_MAX_DISCARD_REQUEST;
1160 dpolicy->io_aware_gran = MAX_PLIST_NUM;
1161 dpolicy->timeout = false;
1162
1163 if (discard_type == DPOLICY_BG) {
1164 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
1165 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
1166 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
1167 dpolicy->io_aware = true;
1168 dpolicy->sync = false;
1169 dpolicy->ordered = true;
1170 if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {
1171 dpolicy->granularity = 1;
1172 if (atomic_read(&dcc->discard_cmd_cnt))
1173 dpolicy->max_interval =
1174 DEF_MIN_DISCARD_ISSUE_TIME;
1175 }
1176 } else if (discard_type == DPOLICY_FORCE) {
1177 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
1178 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
1179 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
1180 dpolicy->io_aware = false;
1181 } else if (discard_type == DPOLICY_FSTRIM) {
1182 dpolicy->io_aware = false;
1183 } else if (discard_type == DPOLICY_UMOUNT) {
1184 dpolicy->io_aware = false;
1185
1186 dpolicy->granularity = 1;
1187 dpolicy->timeout = true;
1188 }
1189}
1190
1191static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1192 struct block_device *bdev, block_t lstart,
1193 block_t start, block_t len);
1194
1195static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
1196 struct discard_policy *dpolicy,
1197 struct discard_cmd *dc,
1198 unsigned int *issued)
1199{
1200 struct block_device *bdev = dc->bdev;
1201 struct request_queue *q = bdev_get_queue(bdev);
1202 unsigned int max_discard_blocks =
1203 SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
1204 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1205 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1206 &(dcc->fstrim_list) : &(dcc->wait_list);
1207 int flag = dpolicy->sync ? REQ_SYNC : 0;
1208 block_t lstart, start, len, total_len;
1209 int err = 0;
1210
1211 if (dc->state != D_PREP)
1212 return 0;
1213
1214 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1215 return 0;
1216
1217 trace_f2fs_issue_discard(bdev, dc->start, dc->len);
1218
1219 lstart = dc->lstart;
1220 start = dc->start;
1221 len = dc->len;
1222 total_len = len;
1223
1224 dc->len = 0;
1225
1226 while (total_len && *issued < dpolicy->max_requests && !err) {
1227 struct bio *bio = NULL;
1228 unsigned long flags;
1229 bool last = true;
1230
1231 if (len > max_discard_blocks) {
1232 len = max_discard_blocks;
1233 last = false;
1234 }
1235
1236 (*issued)++;
1237 if (*issued == dpolicy->max_requests)
1238 last = true;
1239
1240 dc->len += len;
1241
1242 if (time_to_inject(sbi, FAULT_DISCARD)) {
1243 f2fs_show_injection_info(sbi, FAULT_DISCARD);
1244 err = -EIO;
1245 goto submit;
1246 }
1247 err = __blkdev_issue_discard(bdev,
1248 SECTOR_FROM_BLOCK(start),
1249 SECTOR_FROM_BLOCK(len),
1250 GFP_NOFS, 0, &bio);
1251submit:
1252 if (err) {
1253 spin_lock_irqsave(&dc->lock, flags);
1254 if (dc->state == D_PARTIAL)
1255 dc->state = D_SUBMIT;
1256 spin_unlock_irqrestore(&dc->lock, flags);
1257
1258 break;
1259 }
1260
1261 f2fs_bug_on(sbi, !bio);
1262
1263
1264
1265
1266
1267 spin_lock_irqsave(&dc->lock, flags);
1268 if (last)
1269 dc->state = D_SUBMIT;
1270 else
1271 dc->state = D_PARTIAL;
1272 dc->bio_ref++;
1273 spin_unlock_irqrestore(&dc->lock, flags);
1274
1275 atomic_inc(&dcc->queued_discard);
1276 dc->queued++;
1277 list_move_tail(&dc->list, wait_list);
1278
1279
1280 __check_sit_bitmap(sbi, lstart, lstart + len);
1281
1282 bio->bi_private = dc;
1283 bio->bi_end_io = f2fs_submit_discard_endio;
1284 bio->bi_opf |= flag;
1285 submit_bio(bio);
1286
1287 atomic_inc(&dcc->issued_discard);
1288
1289 f2fs_update_iostat(sbi, FS_DISCARD, 1);
1290
1291 lstart += len;
1292 start += len;
1293 total_len -= len;
1294 len = total_len;
1295 }
1296
1297 if (!err && len) {
1298 dcc->undiscard_blks -= len;
1299 __update_discard_tree_range(sbi, bdev, lstart, start, len);
1300 }
1301 return err;
1302}
1303
1304static void __insert_discard_tree(struct f2fs_sb_info *sbi,
1305 struct block_device *bdev, block_t lstart,
1306 block_t start, block_t len,
1307 struct rb_node **insert_p,
1308 struct rb_node *insert_parent)
1309{
1310 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1311 struct rb_node **p;
1312 struct rb_node *parent = NULL;
1313 bool leftmost = true;
1314
1315 if (insert_p && insert_parent) {
1316 parent = insert_parent;
1317 p = insert_p;
1318 goto do_insert;
1319 }
1320
1321 p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,
1322 lstart, &leftmost);
1323do_insert:
1324 __attach_discard_cmd(sbi, bdev, lstart, start, len, parent,
1325 p, leftmost);
1326}
1327
1328static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
1329 struct discard_cmd *dc)
1330{
1331 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);
1332}
1333
1334static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
1335 struct discard_cmd *dc, block_t blkaddr)
1336{
1337 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1338 struct discard_info di = dc->di;
1339 bool modified = false;
1340
1341 if (dc->state == D_DONE || dc->len == 1) {
1342 __remove_discard_cmd(sbi, dc);
1343 return;
1344 }
1345
1346 dcc->undiscard_blks -= di.len;
1347
1348 if (blkaddr > di.lstart) {
1349 dc->len = blkaddr - dc->lstart;
1350 dcc->undiscard_blks += dc->len;
1351 __relocate_discard_cmd(dcc, dc);
1352 modified = true;
1353 }
1354
1355 if (blkaddr < di.lstart + di.len - 1) {
1356 if (modified) {
1357 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
1358 di.start + blkaddr + 1 - di.lstart,
1359 di.lstart + di.len - 1 - blkaddr,
1360 NULL, NULL);
1361 } else {
1362 dc->lstart++;
1363 dc->len--;
1364 dc->start++;
1365 dcc->undiscard_blks += dc->len;
1366 __relocate_discard_cmd(dcc, dc);
1367 }
1368 }
1369}
1370
1371static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1372 struct block_device *bdev, block_t lstart,
1373 block_t start, block_t len)
1374{
1375 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1376 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1377 struct discard_cmd *dc;
1378 struct discard_info di = {0};
1379 struct rb_node **insert_p = NULL, *insert_parent = NULL;
1380 struct request_queue *q = bdev_get_queue(bdev);
1381 unsigned int max_discard_blocks =
1382 SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
1383 block_t end = lstart + len;
1384
1385 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1386 NULL, lstart,
1387 (struct rb_entry **)&prev_dc,
1388 (struct rb_entry **)&next_dc,
1389 &insert_p, &insert_parent, true, NULL);
1390 if (dc)
1391 prev_dc = dc;
1392
1393 if (!prev_dc) {
1394 di.lstart = lstart;
1395 di.len = next_dc ? next_dc->lstart - lstart : len;
1396 di.len = min(di.len, len);
1397 di.start = start;
1398 }
1399
1400 while (1) {
1401 struct rb_node *node;
1402 bool merged = false;
1403 struct discard_cmd *tdc = NULL;
1404
1405 if (prev_dc) {
1406 di.lstart = prev_dc->lstart + prev_dc->len;
1407 if (di.lstart < lstart)
1408 di.lstart = lstart;
1409 if (di.lstart >= end)
1410 break;
1411
1412 if (!next_dc || next_dc->lstart > end)
1413 di.len = end - di.lstart;
1414 else
1415 di.len = next_dc->lstart - di.lstart;
1416 di.start = start + di.lstart - lstart;
1417 }
1418
1419 if (!di.len)
1420 goto next;
1421
1422 if (prev_dc && prev_dc->state == D_PREP &&
1423 prev_dc->bdev == bdev &&
1424 __is_discard_back_mergeable(&di, &prev_dc->di,
1425 max_discard_blocks)) {
1426 prev_dc->di.len += di.len;
1427 dcc->undiscard_blks += di.len;
1428 __relocate_discard_cmd(dcc, prev_dc);
1429 di = prev_dc->di;
1430 tdc = prev_dc;
1431 merged = true;
1432 }
1433
1434 if (next_dc && next_dc->state == D_PREP &&
1435 next_dc->bdev == bdev &&
1436 __is_discard_front_mergeable(&di, &next_dc->di,
1437 max_discard_blocks)) {
1438 next_dc->di.lstart = di.lstart;
1439 next_dc->di.len += di.len;
1440 next_dc->di.start = di.start;
1441 dcc->undiscard_blks += di.len;
1442 __relocate_discard_cmd(dcc, next_dc);
1443 if (tdc)
1444 __remove_discard_cmd(sbi, tdc);
1445 merged = true;
1446 }
1447
1448 if (!merged) {
1449 __insert_discard_tree(sbi, bdev, di.lstart, di.start,
1450 di.len, NULL, NULL);
1451 }
1452 next:
1453 prev_dc = next_dc;
1454 if (!prev_dc)
1455 break;
1456
1457 node = rb_next(&prev_dc->rb_node);
1458 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1459 }
1460}
1461
1462static int __queue_discard_cmd(struct f2fs_sb_info *sbi,
1463 struct block_device *bdev, block_t blkstart, block_t blklen)
1464{
1465 block_t lblkstart = blkstart;
1466
1467 if (!f2fs_bdev_support_discard(bdev))
1468 return 0;
1469
1470 trace_f2fs_queue_discard(bdev, blkstart, blklen);
1471
1472 if (f2fs_is_multi_device(sbi)) {
1473 int devi = f2fs_target_device_index(sbi, blkstart);
1474
1475 blkstart -= FDEV(devi).start_blk;
1476 }
1477 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1478 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
1479 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1480 return 0;
1481}
1482
1483static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
1484 struct discard_policy *dpolicy)
1485{
1486 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1487 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1488 struct rb_node **insert_p = NULL, *insert_parent = NULL;
1489 struct discard_cmd *dc;
1490 struct blk_plug plug;
1491 unsigned int pos = dcc->next_pos;
1492 unsigned int issued = 0;
1493 bool io_interrupted = false;
1494
1495 mutex_lock(&dcc->cmd_lock);
1496 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1497 NULL, pos,
1498 (struct rb_entry **)&prev_dc,
1499 (struct rb_entry **)&next_dc,
1500 &insert_p, &insert_parent, true, NULL);
1501 if (!dc)
1502 dc = next_dc;
1503
1504 blk_start_plug(&plug);
1505
1506 while (dc) {
1507 struct rb_node *node;
1508 int err = 0;
1509
1510 if (dc->state != D_PREP)
1511 goto next;
1512
1513 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
1514 io_interrupted = true;
1515 break;
1516 }
1517
1518 dcc->next_pos = dc->lstart + dc->len;
1519 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1520
1521 if (issued >= dpolicy->max_requests)
1522 break;
1523next:
1524 node = rb_next(&dc->rb_node);
1525 if (err)
1526 __remove_discard_cmd(sbi, dc);
1527 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1528 }
1529
1530 blk_finish_plug(&plug);
1531
1532 if (!dc)
1533 dcc->next_pos = 0;
1534
1535 mutex_unlock(&dcc->cmd_lock);
1536
1537 if (!issued && io_interrupted)
1538 issued = -1;
1539
1540 return issued;
1541}
1542static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1543 struct discard_policy *dpolicy);
1544
1545static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
1546 struct discard_policy *dpolicy)
1547{
1548 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1549 struct list_head *pend_list;
1550 struct discard_cmd *dc, *tmp;
1551 struct blk_plug plug;
1552 int i, issued;
1553 bool io_interrupted = false;
1554
1555 if (dpolicy->timeout)
1556 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);
1557
1558retry:
1559 issued = 0;
1560 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1561 if (dpolicy->timeout &&
1562 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1563 break;
1564
1565 if (i + 1 < dpolicy->granularity)
1566 break;
1567
1568 if (i < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered)
1569 return __issue_discard_cmd_orderly(sbi, dpolicy);
1570
1571 pend_list = &dcc->pend_list[i];
1572
1573 mutex_lock(&dcc->cmd_lock);
1574 if (list_empty(pend_list))
1575 goto next;
1576 if (unlikely(dcc->rbtree_check))
1577 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
1578 &dcc->root, false));
1579 blk_start_plug(&plug);
1580 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1581 f2fs_bug_on(sbi, dc->state != D_PREP);
1582
1583 if (dpolicy->timeout &&
1584 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1585 break;
1586
1587 if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
1588 !is_idle(sbi, DISCARD_TIME)) {
1589 io_interrupted = true;
1590 break;
1591 }
1592
1593 __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1594
1595 if (issued >= dpolicy->max_requests)
1596 break;
1597 }
1598 blk_finish_plug(&plug);
1599next:
1600 mutex_unlock(&dcc->cmd_lock);
1601
1602 if (issued >= dpolicy->max_requests || io_interrupted)
1603 break;
1604 }
1605
1606 if (dpolicy->type == DPOLICY_UMOUNT && issued) {
1607 __wait_all_discard_cmd(sbi, dpolicy);
1608 goto retry;
1609 }
1610
1611 if (!issued && io_interrupted)
1612 issued = -1;
1613
1614 return issued;
1615}
1616
1617static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
1618{
1619 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1620 struct list_head *pend_list;
1621 struct discard_cmd *dc, *tmp;
1622 int i;
1623 bool dropped = false;
1624
1625 mutex_lock(&dcc->cmd_lock);
1626 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1627 pend_list = &dcc->pend_list[i];
1628 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1629 f2fs_bug_on(sbi, dc->state != D_PREP);
1630 __remove_discard_cmd(sbi, dc);
1631 dropped = true;
1632 }
1633 }
1634 mutex_unlock(&dcc->cmd_lock);
1635
1636 return dropped;
1637}
1638
1639void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
1640{
1641 __drop_discard_cmd(sbi);
1642}
1643
1644static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
1645 struct discard_cmd *dc)
1646{
1647 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1648 unsigned int len = 0;
1649
1650 wait_for_completion_io(&dc->wait);
1651 mutex_lock(&dcc->cmd_lock);
1652 f2fs_bug_on(sbi, dc->state != D_DONE);
1653 dc->ref--;
1654 if (!dc->ref) {
1655 if (!dc->error)
1656 len = dc->len;
1657 __remove_discard_cmd(sbi, dc);
1658 }
1659 mutex_unlock(&dcc->cmd_lock);
1660
1661 return len;
1662}
1663
1664static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
1665 struct discard_policy *dpolicy,
1666 block_t start, block_t end)
1667{
1668 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1669 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1670 &(dcc->fstrim_list) : &(dcc->wait_list);
1671 struct discard_cmd *dc, *tmp;
1672 bool need_wait;
1673 unsigned int trimmed = 0;
1674
1675next:
1676 need_wait = false;
1677
1678 mutex_lock(&dcc->cmd_lock);
1679 list_for_each_entry_safe(dc, tmp, wait_list, list) {
1680 if (dc->lstart + dc->len <= start || end <= dc->lstart)
1681 continue;
1682 if (dc->len < dpolicy->granularity)
1683 continue;
1684 if (dc->state == D_DONE && !dc->ref) {
1685 wait_for_completion_io(&dc->wait);
1686 if (!dc->error)
1687 trimmed += dc->len;
1688 __remove_discard_cmd(sbi, dc);
1689 } else {
1690 dc->ref++;
1691 need_wait = true;
1692 break;
1693 }
1694 }
1695 mutex_unlock(&dcc->cmd_lock);
1696
1697 if (need_wait) {
1698 trimmed += __wait_one_discard_bio(sbi, dc);
1699 goto next;
1700 }
1701
1702 return trimmed;
1703}
1704
1705static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1706 struct discard_policy *dpolicy)
1707{
1708 struct discard_policy dp;
1709 unsigned int discard_blks;
1710
1711 if (dpolicy)
1712 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
1713
1714
1715 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1);
1716 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1717 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1);
1718 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1719
1720 return discard_blks;
1721}
1722
1723
1724static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
1725{
1726 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1727 struct discard_cmd *dc;
1728 bool need_wait = false;
1729
1730 mutex_lock(&dcc->cmd_lock);
1731 dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root,
1732 NULL, blkaddr);
1733 if (dc) {
1734 if (dc->state == D_PREP) {
1735 __punch_discard_cmd(sbi, dc, blkaddr);
1736 } else {
1737 dc->ref++;
1738 need_wait = true;
1739 }
1740 }
1741 mutex_unlock(&dcc->cmd_lock);
1742
1743 if (need_wait)
1744 __wait_one_discard_bio(sbi, dc);
1745}
1746
1747void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
1748{
1749 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1750
1751 if (dcc && dcc->f2fs_issue_discard) {
1752 struct task_struct *discard_thread = dcc->f2fs_issue_discard;
1753
1754 dcc->f2fs_issue_discard = NULL;
1755 kthread_stop(discard_thread);
1756 }
1757}
1758
1759
1760bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
1761{
1762 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1763 struct discard_policy dpolicy;
1764 bool dropped;
1765
1766 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
1767 dcc->discard_granularity);
1768 __issue_discard_cmd(sbi, &dpolicy);
1769 dropped = __drop_discard_cmd(sbi);
1770
1771
1772 __wait_all_discard_cmd(sbi, NULL);
1773
1774 f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
1775 return dropped;
1776}
1777
1778static int issue_discard_thread(void *data)
1779{
1780 struct f2fs_sb_info *sbi = data;
1781 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1782 wait_queue_head_t *q = &dcc->discard_wait_queue;
1783 struct discard_policy dpolicy;
1784 unsigned int wait_ms = DEF_MIN_DISCARD_ISSUE_TIME;
1785 int issued;
1786
1787 set_freezable();
1788
1789 do {
1790 if (sbi->gc_mode == GC_URGENT_HIGH ||
1791 !f2fs_available_free_memory(sbi, DISCARD_CACHE))
1792 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);
1793 else
1794 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1795 dcc->discard_granularity);
1796
1797 if (!atomic_read(&dcc->discard_cmd_cnt))
1798 wait_ms = dpolicy.max_interval;
1799
1800 wait_event_interruptible_timeout(*q,
1801 kthread_should_stop() || freezing(current) ||
1802 dcc->discard_wake,
1803 msecs_to_jiffies(wait_ms));
1804
1805 if (dcc->discard_wake)
1806 dcc->discard_wake = 0;
1807
1808
1809 if (atomic_read(&dcc->queued_discard))
1810 __wait_all_discard_cmd(sbi, NULL);
1811
1812 if (try_to_freeze())
1813 continue;
1814 if (f2fs_readonly(sbi->sb))
1815 continue;
1816 if (kthread_should_stop())
1817 return 0;
1818 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
1819 wait_ms = dpolicy.max_interval;
1820 continue;
1821 }
1822 if (!atomic_read(&dcc->discard_cmd_cnt))
1823 continue;
1824
1825 sb_start_intwrite(sbi->sb);
1826
1827 issued = __issue_discard_cmd(sbi, &dpolicy);
1828 if (issued > 0) {
1829 __wait_all_discard_cmd(sbi, &dpolicy);
1830 wait_ms = dpolicy.min_interval;
1831 } else if (issued == -1) {
1832 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
1833 if (!wait_ms)
1834 wait_ms = dpolicy.mid_interval;
1835 } else {
1836 wait_ms = dpolicy.max_interval;
1837 }
1838
1839 sb_end_intwrite(sbi->sb);
1840
1841 } while (!kthread_should_stop());
1842 return 0;
1843}
1844
1845#ifdef CONFIG_BLK_DEV_ZONED
1846static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1847 struct block_device *bdev, block_t blkstart, block_t blklen)
1848{
1849 sector_t sector, nr_sects;
1850 block_t lblkstart = blkstart;
1851 int devi = 0;
1852
1853 if (f2fs_is_multi_device(sbi)) {
1854 devi = f2fs_target_device_index(sbi, blkstart);
1855 if (blkstart < FDEV(devi).start_blk ||
1856 blkstart > FDEV(devi).end_blk) {
1857 f2fs_err(sbi, "Invalid block %x", blkstart);
1858 return -EIO;
1859 }
1860 blkstart -= FDEV(devi).start_blk;
1861 }
1862
1863
1864 if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
1865 sector = SECTOR_FROM_BLOCK(blkstart);
1866 nr_sects = SECTOR_FROM_BLOCK(blklen);
1867
1868 if (sector & (bdev_zone_sectors(bdev) - 1) ||
1869 nr_sects != bdev_zone_sectors(bdev)) {
1870 f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
1871 devi, sbi->s_ndevs ? FDEV(devi).path : "",
1872 blkstart, blklen);
1873 return -EIO;
1874 }
1875 trace_f2fs_issue_reset_zone(bdev, blkstart);
1876 return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
1877 sector, nr_sects, GFP_NOFS);
1878 }
1879
1880
1881 return __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
1882}
1883#endif
1884
1885static int __issue_discard_async(struct f2fs_sb_info *sbi,
1886 struct block_device *bdev, block_t blkstart, block_t blklen)
1887{
1888#ifdef CONFIG_BLK_DEV_ZONED
1889 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
1890 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
1891#endif
1892 return __queue_discard_cmd(sbi, bdev, blkstart, blklen);
1893}
1894
1895static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
1896 block_t blkstart, block_t blklen)
1897{
1898 sector_t start = blkstart, len = 0;
1899 struct block_device *bdev;
1900 struct seg_entry *se;
1901 unsigned int offset;
1902 block_t i;
1903 int err = 0;
1904
1905 bdev = f2fs_target_device(sbi, blkstart, NULL);
1906
1907 for (i = blkstart; i < blkstart + blklen; i++, len++) {
1908 if (i != start) {
1909 struct block_device *bdev2 =
1910 f2fs_target_device(sbi, i, NULL);
1911
1912 if (bdev2 != bdev) {
1913 err = __issue_discard_async(sbi, bdev,
1914 start, len);
1915 if (err)
1916 return err;
1917 bdev = bdev2;
1918 start = i;
1919 len = 0;
1920 }
1921 }
1922
1923 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
1924 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
1925
1926 if (f2fs_block_unit_discard(sbi) &&
1927 !f2fs_test_and_set_bit(offset, se->discard_map))
1928 sbi->discard_blks--;
1929 }
1930
1931 if (len)
1932 err = __issue_discard_async(sbi, bdev, start, len);
1933 return err;
1934}
1935
1936static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
1937 bool check_only)
1938{
1939 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1940 int max_blocks = sbi->blocks_per_seg;
1941 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
1942 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1943 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1944 unsigned long *discard_map = (unsigned long *)se->discard_map;
1945 unsigned long *dmap = SIT_I(sbi)->tmp_map;
1946 unsigned int start = 0, end = -1;
1947 bool force = (cpc->reason & CP_DISCARD);
1948 struct discard_entry *de = NULL;
1949 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
1950 int i;
1951
1952 if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi) ||
1953 !f2fs_block_unit_discard(sbi))
1954 return false;
1955
1956 if (!force) {
1957 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
1958 SM_I(sbi)->dcc_info->nr_discards >=
1959 SM_I(sbi)->dcc_info->max_discards)
1960 return false;
1961 }
1962
1963
1964 for (i = 0; i < entries; i++)
1965 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
1966 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
1967
1968 while (force || SM_I(sbi)->dcc_info->nr_discards <=
1969 SM_I(sbi)->dcc_info->max_discards) {
1970 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
1971 if (start >= max_blocks)
1972 break;
1973
1974 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
1975 if (force && start && end != max_blocks
1976 && (end - start) < cpc->trim_minlen)
1977 continue;
1978
1979 if (check_only)
1980 return true;
1981
1982 if (!de) {
1983 de = f2fs_kmem_cache_alloc(discard_entry_slab,
1984 GFP_F2FS_ZERO, true, NULL);
1985 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
1986 list_add_tail(&de->list, head);
1987 }
1988
1989 for (i = start; i < end; i++)
1990 __set_bit_le(i, (void *)de->discard_map);
1991
1992 SM_I(sbi)->dcc_info->nr_discards += end - start;
1993 }
1994 return false;
1995}
1996
1997static void release_discard_addr(struct discard_entry *entry)
1998{
1999 list_del(&entry->list);
2000 kmem_cache_free(discard_entry_slab, entry);
2001}
2002
2003void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
2004{
2005 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
2006 struct discard_entry *entry, *this;
2007
2008
2009 list_for_each_entry_safe(entry, this, head, list)
2010 release_discard_addr(entry);
2011}
2012
2013
2014
2015
2016static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
2017{
2018 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2019 unsigned int segno;
2020
2021 mutex_lock(&dirty_i->seglist_lock);
2022 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
2023 __set_test_and_free(sbi, segno, false);
2024 mutex_unlock(&dirty_i->seglist_lock);
2025}
2026
2027void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
2028 struct cp_control *cpc)
2029{
2030 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2031 struct list_head *head = &dcc->entry_list;
2032 struct discard_entry *entry, *this;
2033 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2034 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
2035 unsigned int start = 0, end = -1;
2036 unsigned int secno, start_segno;
2037 bool force = (cpc->reason & CP_DISCARD);
2038 bool section_alignment = F2FS_OPTION(sbi).discard_unit ==
2039 DISCARD_UNIT_SECTION;
2040
2041 if (f2fs_lfs_mode(sbi) && __is_large_section(sbi))
2042 section_alignment = true;
2043
2044 mutex_lock(&dirty_i->seglist_lock);
2045
2046 while (1) {
2047 int i;
2048
2049 if (section_alignment && end != -1)
2050 end--;
2051 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
2052 if (start >= MAIN_SEGS(sbi))
2053 break;
2054 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
2055 start + 1);
2056
2057 if (section_alignment) {
2058 start = rounddown(start, sbi->segs_per_sec);
2059 end = roundup(end, sbi->segs_per_sec);
2060 }
2061
2062 for (i = start; i < end; i++) {
2063 if (test_and_clear_bit(i, prefree_map))
2064 dirty_i->nr_dirty[PRE]--;
2065 }
2066
2067 if (!f2fs_realtime_discard_enable(sbi))
2068 continue;
2069
2070 if (force && start >= cpc->trim_start &&
2071 (end - 1) <= cpc->trim_end)
2072 continue;
2073
2074 if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) {
2075 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
2076 (end - start) << sbi->log_blocks_per_seg);
2077 continue;
2078 }
2079next:
2080 secno = GET_SEC_FROM_SEG(sbi, start);
2081 start_segno = GET_SEG_FROM_SEC(sbi, secno);
2082 if (!IS_CURSEC(sbi, secno) &&
2083 !get_valid_blocks(sbi, start, true))
2084 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
2085 sbi->segs_per_sec << sbi->log_blocks_per_seg);
2086
2087 start = start_segno + sbi->segs_per_sec;
2088 if (start < end)
2089 goto next;
2090 else
2091 end = start - 1;
2092 }
2093 mutex_unlock(&dirty_i->seglist_lock);
2094
2095 if (!f2fs_block_unit_discard(sbi))
2096 goto wakeup;
2097
2098
2099 list_for_each_entry_safe(entry, this, head, list) {
2100 unsigned int cur_pos = 0, next_pos, len, total_len = 0;
2101 bool is_valid = test_bit_le(0, entry->discard_map);
2102
2103find_next:
2104 if (is_valid) {
2105 next_pos = find_next_zero_bit_le(entry->discard_map,
2106 sbi->blocks_per_seg, cur_pos);
2107 len = next_pos - cur_pos;
2108
2109 if (f2fs_sb_has_blkzoned(sbi) ||
2110 (force && len < cpc->trim_minlen))
2111 goto skip;
2112
2113 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
2114 len);
2115 total_len += len;
2116 } else {
2117 next_pos = find_next_bit_le(entry->discard_map,
2118 sbi->blocks_per_seg, cur_pos);
2119 }
2120skip:
2121 cur_pos = next_pos;
2122 is_valid = !is_valid;
2123
2124 if (cur_pos < sbi->blocks_per_seg)
2125 goto find_next;
2126
2127 release_discard_addr(entry);
2128 dcc->nr_discards -= total_len;
2129 }
2130
2131wakeup:
2132 wake_up_discard_thread(sbi, false);
2133}
2134
2135int f2fs_start_discard_thread(struct f2fs_sb_info *sbi)
2136{
2137 dev_t dev = sbi->sb->s_bdev->bd_dev;
2138 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2139 int err = 0;
2140
2141 if (!f2fs_realtime_discard_enable(sbi))
2142 return 0;
2143
2144 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2145 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2146 if (IS_ERR(dcc->f2fs_issue_discard))
2147 err = PTR_ERR(dcc->f2fs_issue_discard);
2148
2149 return err;
2150}
2151
2152static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2153{
2154 struct discard_cmd_control *dcc;
2155 int err = 0, i;
2156
2157 if (SM_I(sbi)->dcc_info) {
2158 dcc = SM_I(sbi)->dcc_info;
2159 goto init_thread;
2160 }
2161
2162 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2163 if (!dcc)
2164 return -ENOMEM;
2165
2166 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2167 if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT)
2168 dcc->discard_granularity = sbi->blocks_per_seg;
2169 else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION)
2170 dcc->discard_granularity = BLKS_PER_SEC(sbi);
2171
2172 INIT_LIST_HEAD(&dcc->entry_list);
2173 for (i = 0; i < MAX_PLIST_NUM; i++)
2174 INIT_LIST_HEAD(&dcc->pend_list[i]);
2175 INIT_LIST_HEAD(&dcc->wait_list);
2176 INIT_LIST_HEAD(&dcc->fstrim_list);
2177 mutex_init(&dcc->cmd_lock);
2178 atomic_set(&dcc->issued_discard, 0);
2179 atomic_set(&dcc->queued_discard, 0);
2180 atomic_set(&dcc->discard_cmd_cnt, 0);
2181 dcc->nr_discards = 0;
2182 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
2183 dcc->undiscard_blks = 0;
2184 dcc->next_pos = 0;
2185 dcc->root = RB_ROOT_CACHED;
2186 dcc->rbtree_check = false;
2187
2188 init_waitqueue_head(&dcc->discard_wait_queue);
2189 SM_I(sbi)->dcc_info = dcc;
2190init_thread:
2191 err = f2fs_start_discard_thread(sbi);
2192 if (err) {
2193 kfree(dcc);
2194 SM_I(sbi)->dcc_info = NULL;
2195 }
2196
2197 return err;
2198}
2199
2200static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2201{
2202 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2203
2204 if (!dcc)
2205 return;
2206
2207 f2fs_stop_discard_thread(sbi);
2208
2209
2210
2211
2212
2213 if (unlikely(atomic_read(&dcc->discard_cmd_cnt)))
2214 f2fs_issue_discard_timeout(sbi);
2215
2216 kfree(dcc);
2217 SM_I(sbi)->dcc_info = NULL;
2218}
2219
2220static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
2221{
2222 struct sit_info *sit_i = SIT_I(sbi);
2223
2224 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
2225 sit_i->dirty_sentries++;
2226 return false;
2227 }
2228
2229 return true;
2230}
2231
2232static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
2233 unsigned int segno, int modified)
2234{
2235 struct seg_entry *se = get_seg_entry(sbi, segno);
2236
2237 se->type = type;
2238 if (modified)
2239 __mark_sit_entry_dirty(sbi, segno);
2240}
2241
2242static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
2243 block_t blkaddr)
2244{
2245 unsigned int segno = GET_SEGNO(sbi, blkaddr);
2246
2247 if (segno == NULL_SEGNO)
2248 return 0;
2249 return get_seg_entry(sbi, segno)->mtime;
2250}
2251
2252static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
2253 unsigned long long old_mtime)
2254{
2255 struct seg_entry *se;
2256 unsigned int segno = GET_SEGNO(sbi, blkaddr);
2257 unsigned long long ctime = get_mtime(sbi, false);
2258 unsigned long long mtime = old_mtime ? old_mtime : ctime;
2259
2260 if (segno == NULL_SEGNO)
2261 return;
2262
2263 se = get_seg_entry(sbi, segno);
2264
2265 if (!se->mtime)
2266 se->mtime = mtime;
2267 else
2268 se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
2269 se->valid_blocks + 1);
2270
2271 if (ctime > SIT_I(sbi)->max_mtime)
2272 SIT_I(sbi)->max_mtime = ctime;
2273}
2274
2275static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
2276{
2277 struct seg_entry *se;
2278 unsigned int segno, offset;
2279 long int new_vblocks;
2280 bool exist;
2281#ifdef CONFIG_F2FS_CHECK_FS
2282 bool mir_exist;
2283#endif
2284
2285 segno = GET_SEGNO(sbi, blkaddr);
2286
2287 se = get_seg_entry(sbi, segno);
2288 new_vblocks = se->valid_blocks + del;
2289 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2290
2291 f2fs_bug_on(sbi, (new_vblocks < 0 ||
2292 (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
2293
2294 se->valid_blocks = new_vblocks;
2295
2296
2297 if (del > 0) {
2298 exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
2299#ifdef CONFIG_F2FS_CHECK_FS
2300 mir_exist = f2fs_test_and_set_bit(offset,
2301 se->cur_valid_map_mir);
2302 if (unlikely(exist != mir_exist)) {
2303 f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
2304 blkaddr, exist);
2305 f2fs_bug_on(sbi, 1);
2306 }
2307#endif
2308 if (unlikely(exist)) {
2309 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
2310 blkaddr);
2311 f2fs_bug_on(sbi, 1);
2312 se->valid_blocks--;
2313 del = 0;
2314 }
2315
2316 if (f2fs_block_unit_discard(sbi) &&
2317 !f2fs_test_and_set_bit(offset, se->discard_map))
2318 sbi->discard_blks--;
2319
2320
2321
2322
2323
2324 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2325 if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
2326 se->ckpt_valid_blocks++;
2327 }
2328 } else {
2329 exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);
2330#ifdef CONFIG_F2FS_CHECK_FS
2331 mir_exist = f2fs_test_and_clear_bit(offset,
2332 se->cur_valid_map_mir);
2333 if (unlikely(exist != mir_exist)) {
2334 f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
2335 blkaddr, exist);
2336 f2fs_bug_on(sbi, 1);
2337 }
2338#endif
2339 if (unlikely(!exist)) {
2340 f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",
2341 blkaddr);
2342 f2fs_bug_on(sbi, 1);
2343 se->valid_blocks++;
2344 del = 0;
2345 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2346
2347
2348
2349
2350
2351
2352 if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
2353 spin_lock(&sbi->stat_lock);
2354 sbi->unusable_block_count++;
2355 spin_unlock(&sbi->stat_lock);
2356 }
2357 }
2358
2359 if (f2fs_block_unit_discard(sbi) &&
2360 f2fs_test_and_clear_bit(offset, se->discard_map))
2361 sbi->discard_blks++;
2362 }
2363 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2364 se->ckpt_valid_blocks += del;
2365
2366 __mark_sit_entry_dirty(sbi, segno);
2367
2368
2369 SIT_I(sbi)->written_valid_blocks += del;
2370
2371 if (__is_large_section(sbi))
2372 get_sec_entry(sbi, segno)->valid_blocks += del;
2373}
2374
2375void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
2376{
2377 unsigned int segno = GET_SEGNO(sbi, addr);
2378 struct sit_info *sit_i = SIT_I(sbi);
2379
2380 f2fs_bug_on(sbi, addr == NULL_ADDR);
2381 if (addr == NEW_ADDR || addr == COMPRESS_ADDR)
2382 return;
2383
2384 invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);
2385 f2fs_invalidate_compress_page(sbi, addr);
2386
2387
2388 down_write(&sit_i->sentry_lock);
2389
2390 update_segment_mtime(sbi, addr, 0);
2391 update_sit_entry(sbi, addr, -1);
2392
2393
2394 locate_dirty_segment(sbi, segno);
2395
2396 up_write(&sit_i->sentry_lock);
2397}
2398
2399bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
2400{
2401 struct sit_info *sit_i = SIT_I(sbi);
2402 unsigned int segno, offset;
2403 struct seg_entry *se;
2404 bool is_cp = false;
2405
2406 if (!__is_valid_data_blkaddr(blkaddr))
2407 return true;
2408
2409 down_read(&sit_i->sentry_lock);
2410
2411 segno = GET_SEGNO(sbi, blkaddr);
2412 se = get_seg_entry(sbi, segno);
2413 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2414
2415 if (f2fs_test_bit(offset, se->ckpt_valid_map))
2416 is_cp = true;
2417
2418 up_read(&sit_i->sentry_lock);
2419
2420 return is_cp;
2421}
2422
2423
2424
2425
2426static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
2427 struct f2fs_summary *sum)
2428{
2429 struct curseg_info *curseg = CURSEG_I(sbi, type);
2430 void *addr = curseg->sum_blk;
2431
2432 addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
2433 memcpy(addr, sum, sizeof(struct f2fs_summary));
2434}
2435
2436
2437
2438
2439int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
2440{
2441 int valid_sum_count = 0;
2442 int i, sum_in_page;
2443
2444 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2445 if (sbi->ckpt->alloc_type[i] == SSR)
2446 valid_sum_count += sbi->blocks_per_seg;
2447 else {
2448 if (for_ra)
2449 valid_sum_count += le16_to_cpu(
2450 F2FS_CKPT(sbi)->cur_data_blkoff[i]);
2451 else
2452 valid_sum_count += curseg_blkoff(sbi, i);
2453 }
2454 }
2455
2456 sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
2457 SUM_FOOTER_SIZE) / SUMMARY_SIZE;
2458 if (valid_sum_count <= sum_in_page)
2459 return 1;
2460 else if ((valid_sum_count - sum_in_page) <=
2461 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
2462 return 2;
2463 return 3;
2464}
2465
2466
2467
2468
2469struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
2470{
2471 if (unlikely(f2fs_cp_error(sbi)))
2472 return ERR_PTR(-EIO);
2473 return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));
2474}
2475
2476void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
2477 void *src, block_t blk_addr)
2478{
2479 struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2480
2481 memcpy(page_address(page), src, PAGE_SIZE);
2482 set_page_dirty(page);
2483 f2fs_put_page(page, 1);
2484}
2485
2486static void write_sum_page(struct f2fs_sb_info *sbi,
2487 struct f2fs_summary_block *sum_blk, block_t blk_addr)
2488{
2489 f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
2490}
2491
2492static void write_current_sum_page(struct f2fs_sb_info *sbi,
2493 int type, block_t blk_addr)
2494{
2495 struct curseg_info *curseg = CURSEG_I(sbi, type);
2496 struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2497 struct f2fs_summary_block *src = curseg->sum_blk;
2498 struct f2fs_summary_block *dst;
2499
2500 dst = (struct f2fs_summary_block *)page_address(page);
2501 memset(dst, 0, PAGE_SIZE);
2502
2503 mutex_lock(&curseg->curseg_mutex);
2504
2505 down_read(&curseg->journal_rwsem);
2506 memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
2507 up_read(&curseg->journal_rwsem);
2508
2509 memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
2510 memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
2511
2512 mutex_unlock(&curseg->curseg_mutex);
2513
2514 set_page_dirty(page);
2515 f2fs_put_page(page, 1);
2516}
2517
2518static int is_next_segment_free(struct f2fs_sb_info *sbi,
2519 struct curseg_info *curseg, int type)
2520{
2521 unsigned int segno = curseg->segno + 1;
2522 struct free_segmap_info *free_i = FREE_I(sbi);
2523
2524 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
2525 return !test_bit(segno, free_i->free_segmap);
2526 return 0;
2527}
2528
2529
2530
2531
2532
2533static void get_new_segment(struct f2fs_sb_info *sbi,
2534 unsigned int *newseg, bool new_sec, int dir)
2535{
2536 struct free_segmap_info *free_i = FREE_I(sbi);
2537 unsigned int segno, secno, zoneno;
2538 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
2539 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
2540 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
2541 unsigned int left_start = hint;
2542 bool init = true;
2543 int go_left = 0;
2544 int i;
2545
2546 spin_lock(&free_i->segmap_lock);
2547
2548 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
2549 segno = find_next_zero_bit(free_i->free_segmap,
2550 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
2551 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
2552 goto got_it;
2553 }
2554find_other_zone:
2555 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
2556 if (secno >= MAIN_SECS(sbi)) {
2557 if (dir == ALLOC_RIGHT) {
2558 secno = find_first_zero_bit(free_i->free_secmap,
2559 MAIN_SECS(sbi));
2560 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
2561 } else {
2562 go_left = 1;
2563 left_start = hint - 1;
2564 }
2565 }
2566 if (go_left == 0)
2567 goto skip_left;
2568
2569 while (test_bit(left_start, free_i->free_secmap)) {
2570 if (left_start > 0) {
2571 left_start--;
2572 continue;
2573 }
2574 left_start = find_first_zero_bit(free_i->free_secmap,
2575 MAIN_SECS(sbi));
2576 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
2577 break;
2578 }
2579 secno = left_start;
2580skip_left:
2581 segno = GET_SEG_FROM_SEC(sbi, secno);
2582 zoneno = GET_ZONE_FROM_SEC(sbi, secno);
2583
2584
2585 if (!init)
2586 goto got_it;
2587 if (sbi->secs_per_zone == 1)
2588 goto got_it;
2589 if (zoneno == old_zoneno)
2590 goto got_it;
2591 if (dir == ALLOC_LEFT) {
2592 if (!go_left && zoneno + 1 >= total_zones)
2593 goto got_it;
2594 if (go_left && zoneno == 0)
2595 goto got_it;
2596 }
2597 for (i = 0; i < NR_CURSEG_TYPE; i++)
2598 if (CURSEG_I(sbi, i)->zone == zoneno)
2599 break;
2600
2601 if (i < NR_CURSEG_TYPE) {
2602
2603 if (go_left)
2604 hint = zoneno * sbi->secs_per_zone - 1;
2605 else if (zoneno + 1 >= total_zones)
2606 hint = 0;
2607 else
2608 hint = (zoneno + 1) * sbi->secs_per_zone;
2609 init = false;
2610 goto find_other_zone;
2611 }
2612got_it:
2613
2614 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
2615 __set_inuse(sbi, segno);
2616 *newseg = segno;
2617 spin_unlock(&free_i->segmap_lock);
2618}
2619
2620static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
2621{
2622 struct curseg_info *curseg = CURSEG_I(sbi, type);
2623 struct summary_footer *sum_footer;
2624 unsigned short seg_type = curseg->seg_type;
2625
2626 curseg->inited = true;
2627 curseg->segno = curseg->next_segno;
2628 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
2629 curseg->next_blkoff = 0;
2630 curseg->next_segno = NULL_SEGNO;
2631
2632 sum_footer = &(curseg->sum_blk->footer);
2633 memset(sum_footer, 0, sizeof(struct summary_footer));
2634
2635 sanity_check_seg_type(sbi, seg_type);
2636
2637 if (IS_DATASEG(seg_type))
2638 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
2639 if (IS_NODESEG(seg_type))
2640 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
2641 __set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
2642}
2643
2644static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
2645{
2646 struct curseg_info *curseg = CURSEG_I(sbi, type);
2647 unsigned short seg_type = curseg->seg_type;
2648
2649 sanity_check_seg_type(sbi, seg_type);
2650 if (f2fs_need_rand_seg(sbi))
2651 return prandom_u32() % (MAIN_SECS(sbi) * sbi->segs_per_sec);
2652
2653
2654 if (__is_large_section(sbi))
2655 return curseg->segno;
2656
2657
2658 if (!curseg->inited)
2659 return 0;
2660
2661 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2662 return 0;
2663
2664 if (test_opt(sbi, NOHEAP) &&
2665 (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type)))
2666 return 0;
2667
2668 if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
2669 return SIT_I(sbi)->last_victim[ALLOC_NEXT];
2670
2671
2672 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
2673 return 0;
2674
2675 return curseg->segno;
2676}
2677
2678
2679
2680
2681
2682static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
2683{
2684 struct curseg_info *curseg = CURSEG_I(sbi, type);
2685 unsigned short seg_type = curseg->seg_type;
2686 unsigned int segno = curseg->segno;
2687 int dir = ALLOC_LEFT;
2688
2689 if (curseg->inited)
2690 write_sum_page(sbi, curseg->sum_blk,
2691 GET_SUM_BLOCK(sbi, segno));
2692 if (seg_type == CURSEG_WARM_DATA || seg_type == CURSEG_COLD_DATA)
2693 dir = ALLOC_RIGHT;
2694
2695 if (test_opt(sbi, NOHEAP))
2696 dir = ALLOC_RIGHT;
2697
2698 segno = __get_next_segno(sbi, type);
2699 get_new_segment(sbi, &segno, new_sec, dir);
2700 curseg->next_segno = segno;
2701 reset_curseg(sbi, type, 1);
2702 curseg->alloc_type = LFS;
2703 if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)
2704 curseg->fragment_remained_chunk =
2705 prandom_u32() % sbi->max_fragment_chunk + 1;
2706}
2707
2708static int __next_free_blkoff(struct f2fs_sb_info *sbi,
2709 int segno, block_t start)
2710{
2711 struct seg_entry *se = get_seg_entry(sbi, segno);
2712 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2713 unsigned long *target_map = SIT_I(sbi)->tmp_map;
2714 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2715 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2716 int i;
2717
2718 for (i = 0; i < entries; i++)
2719 target_map[i] = ckpt_map[i] | cur_map[i];
2720
2721 return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
2722}
2723
2724
2725
2726
2727
2728
2729static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
2730 struct curseg_info *seg)
2731{
2732 if (seg->alloc_type == SSR) {
2733 seg->next_blkoff =
2734 __next_free_blkoff(sbi, seg->segno,
2735 seg->next_blkoff + 1);
2736 } else {
2737 seg->next_blkoff++;
2738 if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) {
2739
2740 if (--seg->fragment_remained_chunk <= 0) {
2741 seg->fragment_remained_chunk =
2742 prandom_u32() % sbi->max_fragment_chunk + 1;
2743 seg->next_blkoff +=
2744 prandom_u32() % sbi->max_fragment_hole + 1;
2745 }
2746 }
2747 }
2748}
2749
2750bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
2751{
2752 return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg;
2753}
2754
2755
2756
2757
2758
2759static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush)
2760{
2761 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2762 struct curseg_info *curseg = CURSEG_I(sbi, type);
2763 unsigned int new_segno = curseg->next_segno;
2764 struct f2fs_summary_block *sum_node;
2765 struct page *sum_page;
2766
2767 if (flush)
2768 write_sum_page(sbi, curseg->sum_blk,
2769 GET_SUM_BLOCK(sbi, curseg->segno));
2770
2771 __set_test_and_inuse(sbi, new_segno);
2772
2773 mutex_lock(&dirty_i->seglist_lock);
2774 __remove_dirty_segment(sbi, new_segno, PRE);
2775 __remove_dirty_segment(sbi, new_segno, DIRTY);
2776 mutex_unlock(&dirty_i->seglist_lock);
2777
2778 reset_curseg(sbi, type, 1);
2779 curseg->alloc_type = SSR;
2780 curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
2781
2782 sum_page = f2fs_get_sum_page(sbi, new_segno);
2783 if (IS_ERR(sum_page)) {
2784
2785 memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE);
2786 return;
2787 }
2788 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
2789 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
2790 f2fs_put_page(sum_page, 1);
2791}
2792
2793static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2794 int alloc_mode, unsigned long long age);
2795
2796static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,
2797 int target_type, int alloc_mode,
2798 unsigned long long age)
2799{
2800 struct curseg_info *curseg = CURSEG_I(sbi, type);
2801
2802 curseg->seg_type = target_type;
2803
2804 if (get_ssr_segment(sbi, type, alloc_mode, age)) {
2805 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
2806
2807 curseg->seg_type = se->type;
2808 change_curseg(sbi, type, true);
2809 } else {
2810
2811 curseg->seg_type = CURSEG_COLD_DATA;
2812 new_curseg(sbi, type, true);
2813 }
2814 stat_inc_seg_type(sbi, curseg);
2815}
2816
2817static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)
2818{
2819 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
2820
2821 if (!sbi->am.atgc_enabled)
2822 return;
2823
2824 down_read(&SM_I(sbi)->curseg_lock);
2825
2826 mutex_lock(&curseg->curseg_mutex);
2827 down_write(&SIT_I(sbi)->sentry_lock);
2828
2829 get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);
2830
2831 up_write(&SIT_I(sbi)->sentry_lock);
2832 mutex_unlock(&curseg->curseg_mutex);
2833
2834 up_read(&SM_I(sbi)->curseg_lock);
2835
2836}
2837void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
2838{
2839 __f2fs_init_atgc_curseg(sbi);
2840}
2841
2842static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2843{
2844 struct curseg_info *curseg = CURSEG_I(sbi, type);
2845
2846 mutex_lock(&curseg->curseg_mutex);
2847 if (!curseg->inited)
2848 goto out;
2849
2850 if (get_valid_blocks(sbi, curseg->segno, false)) {
2851 write_sum_page(sbi, curseg->sum_blk,
2852 GET_SUM_BLOCK(sbi, curseg->segno));
2853 } else {
2854 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2855 __set_test_and_free(sbi, curseg->segno, true);
2856 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2857 }
2858out:
2859 mutex_unlock(&curseg->curseg_mutex);
2860}
2861
2862void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
2863{
2864 __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2865
2866 if (sbi->am.atgc_enabled)
2867 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2868}
2869
2870static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2871{
2872 struct curseg_info *curseg = CURSEG_I(sbi, type);
2873
2874 mutex_lock(&curseg->curseg_mutex);
2875 if (!curseg->inited)
2876 goto out;
2877 if (get_valid_blocks(sbi, curseg->segno, false))
2878 goto out;
2879
2880 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2881 __set_test_and_inuse(sbi, curseg->segno);
2882 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2883out:
2884 mutex_unlock(&curseg->curseg_mutex);
2885}
2886
2887void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
2888{
2889 __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2890
2891 if (sbi->am.atgc_enabled)
2892 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2893}
2894
2895static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2896 int alloc_mode, unsigned long long age)
2897{
2898 struct curseg_info *curseg = CURSEG_I(sbi, type);
2899 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
2900 unsigned segno = NULL_SEGNO;
2901 unsigned short seg_type = curseg->seg_type;
2902 int i, cnt;
2903 bool reversed = false;
2904
2905 sanity_check_seg_type(sbi, seg_type);
2906
2907
2908 if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {
2909 curseg->next_segno = segno;
2910 return 1;
2911 }
2912
2913
2914 if (IS_NODESEG(seg_type)) {
2915 if (seg_type >= CURSEG_WARM_NODE) {
2916 reversed = true;
2917 i = CURSEG_COLD_NODE;
2918 } else {
2919 i = CURSEG_HOT_NODE;
2920 }
2921 cnt = NR_CURSEG_NODE_TYPE;
2922 } else {
2923 if (seg_type >= CURSEG_WARM_DATA) {
2924 reversed = true;
2925 i = CURSEG_COLD_DATA;
2926 } else {
2927 i = CURSEG_HOT_DATA;
2928 }
2929 cnt = NR_CURSEG_DATA_TYPE;
2930 }
2931
2932 for (; cnt-- > 0; reversed ? i-- : i++) {
2933 if (i == seg_type)
2934 continue;
2935 if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {
2936 curseg->next_segno = segno;
2937 return 1;
2938 }
2939 }
2940
2941
2942 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2943 segno = get_free_segment(sbi);
2944 if (segno != NULL_SEGNO) {
2945 curseg->next_segno = segno;
2946 return 1;
2947 }
2948 }
2949 return 0;
2950}
2951
2952
2953
2954
2955
2956static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
2957 int type, bool force)
2958{
2959 struct curseg_info *curseg = CURSEG_I(sbi, type);
2960
2961 if (force)
2962 new_curseg(sbi, type, true);
2963 else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
2964 curseg->seg_type == CURSEG_WARM_NODE)
2965 new_curseg(sbi, type, false);
2966 else if (curseg->alloc_type == LFS &&
2967 is_next_segment_free(sbi, curseg, type) &&
2968 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2969 new_curseg(sbi, type, false);
2970 else if (f2fs_need_SSR(sbi) &&
2971 get_ssr_segment(sbi, type, SSR, 0))
2972 change_curseg(sbi, type, true);
2973 else
2974 new_curseg(sbi, type, false);
2975
2976 stat_inc_seg_type(sbi, curseg);
2977}
2978
2979void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
2980 unsigned int start, unsigned int end)
2981{
2982 struct curseg_info *curseg = CURSEG_I(sbi, type);
2983 unsigned int segno;
2984
2985 down_read(&SM_I(sbi)->curseg_lock);
2986 mutex_lock(&curseg->curseg_mutex);
2987 down_write(&SIT_I(sbi)->sentry_lock);
2988
2989 segno = CURSEG_I(sbi, type)->segno;
2990 if (segno < start || segno > end)
2991 goto unlock;
2992
2993 if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
2994 change_curseg(sbi, type, true);
2995 else
2996 new_curseg(sbi, type, true);
2997
2998 stat_inc_seg_type(sbi, curseg);
2999
3000 locate_dirty_segment(sbi, segno);
3001unlock:
3002 up_write(&SIT_I(sbi)->sentry_lock);
3003
3004 if (segno != curseg->segno)
3005 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
3006 type, segno, curseg->segno);
3007
3008 mutex_unlock(&curseg->curseg_mutex);
3009 up_read(&SM_I(sbi)->curseg_lock);
3010}
3011
3012static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
3013 bool new_sec, bool force)
3014{
3015 struct curseg_info *curseg = CURSEG_I(sbi, type);
3016 unsigned int old_segno;
3017
3018 if (!curseg->inited)
3019 goto alloc;
3020
3021 if (force || curseg->next_blkoff ||
3022 get_valid_blocks(sbi, curseg->segno, new_sec))
3023 goto alloc;
3024
3025 if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
3026 return;
3027alloc:
3028 old_segno = curseg->segno;
3029 SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
3030 locate_dirty_segment(sbi, old_segno);
3031}
3032
3033static void __allocate_new_section(struct f2fs_sb_info *sbi,
3034 int type, bool force)
3035{
3036 __allocate_new_segment(sbi, type, true, force);
3037}
3038
3039void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
3040{
3041 down_read(&SM_I(sbi)->curseg_lock);
3042 down_write(&SIT_I(sbi)->sentry_lock);
3043 __allocate_new_section(sbi, type, force);
3044 up_write(&SIT_I(sbi)->sentry_lock);
3045 up_read(&SM_I(sbi)->curseg_lock);
3046}
3047
3048void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
3049{
3050 int i;
3051
3052 down_read(&SM_I(sbi)->curseg_lock);
3053 down_write(&SIT_I(sbi)->sentry_lock);
3054 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
3055 __allocate_new_segment(sbi, i, false, false);
3056 up_write(&SIT_I(sbi)->sentry_lock);
3057 up_read(&SM_I(sbi)->curseg_lock);
3058}
3059
3060static const struct segment_allocation default_salloc_ops = {
3061 .allocate_segment = allocate_segment_by_default,
3062};
3063
3064bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
3065 struct cp_control *cpc)
3066{
3067 __u64 trim_start = cpc->trim_start;
3068 bool has_candidate = false;
3069
3070 down_write(&SIT_I(sbi)->sentry_lock);
3071 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
3072 if (add_discard_addrs(sbi, cpc, true)) {
3073 has_candidate = true;
3074 break;
3075 }
3076 }
3077 up_write(&SIT_I(sbi)->sentry_lock);
3078
3079 cpc->trim_start = trim_start;
3080 return has_candidate;
3081}
3082
3083static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
3084 struct discard_policy *dpolicy,
3085 unsigned int start, unsigned int end)
3086{
3087 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
3088 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
3089 struct rb_node **insert_p = NULL, *insert_parent = NULL;
3090 struct discard_cmd *dc;
3091 struct blk_plug plug;
3092 int issued;
3093 unsigned int trimmed = 0;
3094
3095next:
3096 issued = 0;
3097
3098 mutex_lock(&dcc->cmd_lock);
3099 if (unlikely(dcc->rbtree_check))
3100 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
3101 &dcc->root, false));
3102
3103 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
3104 NULL, start,
3105 (struct rb_entry **)&prev_dc,
3106 (struct rb_entry **)&next_dc,
3107 &insert_p, &insert_parent, true, NULL);
3108 if (!dc)
3109 dc = next_dc;
3110
3111 blk_start_plug(&plug);
3112
3113 while (dc && dc->lstart <= end) {
3114 struct rb_node *node;
3115 int err = 0;
3116
3117 if (dc->len < dpolicy->granularity)
3118 goto skip;
3119
3120 if (dc->state != D_PREP) {
3121 list_move_tail(&dc->list, &dcc->fstrim_list);
3122 goto skip;
3123 }
3124
3125 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
3126
3127 if (issued >= dpolicy->max_requests) {
3128 start = dc->lstart + dc->len;
3129
3130 if (err)
3131 __remove_discard_cmd(sbi, dc);
3132
3133 blk_finish_plug(&plug);
3134 mutex_unlock(&dcc->cmd_lock);
3135 trimmed += __wait_all_discard_cmd(sbi, NULL);
3136 congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
3137 goto next;
3138 }
3139skip:
3140 node = rb_next(&dc->rb_node);
3141 if (err)
3142 __remove_discard_cmd(sbi, dc);
3143 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
3144
3145 if (fatal_signal_pending(current))
3146 break;
3147 }
3148
3149 blk_finish_plug(&plug);
3150 mutex_unlock(&dcc->cmd_lock);
3151
3152 return trimmed;
3153}
3154
3155int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
3156{
3157 __u64 start = F2FS_BYTES_TO_BLK(range->start);
3158 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
3159 unsigned int start_segno, end_segno;
3160 block_t start_block, end_block;
3161 struct cp_control cpc;
3162 struct discard_policy dpolicy;
3163 unsigned long long trimmed = 0;
3164 int err = 0;
3165 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
3166
3167 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
3168 return -EINVAL;
3169
3170 if (end < MAIN_BLKADDR(sbi))
3171 goto out;
3172
3173 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
3174 f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
3175 return -EFSCORRUPTED;
3176 }
3177
3178
3179 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
3180 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
3181 GET_SEGNO(sbi, end);
3182 if (need_align) {
3183 start_segno = rounddown(start_segno, sbi->segs_per_sec);
3184 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;
3185 }
3186
3187 cpc.reason = CP_DISCARD;
3188 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
3189 cpc.trim_start = start_segno;
3190 cpc.trim_end = end_segno;
3191
3192 if (sbi->discard_blks == 0)
3193 goto out;
3194
3195 down_write(&sbi->gc_lock);
3196 err = f2fs_write_checkpoint(sbi, &cpc);
3197 up_write(&sbi->gc_lock);
3198 if (err)
3199 goto out;
3200
3201
3202
3203
3204
3205
3206
3207 if (f2fs_realtime_discard_enable(sbi))
3208 goto out;
3209
3210 start_block = START_BLOCK(sbi, start_segno);
3211 end_block = START_BLOCK(sbi, end_segno + 1);
3212
3213 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
3214 trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
3215 start_block, end_block);
3216
3217 trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
3218 start_block, end_block);
3219out:
3220 if (!err)
3221 range->len = F2FS_BLK_TO_BYTES(trimmed);
3222 return err;
3223}
3224
3225static bool __has_curseg_space(struct f2fs_sb_info *sbi,
3226 struct curseg_info *curseg)
3227{
3228 return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi,
3229 curseg->segno);
3230}
3231
3232int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
3233{
3234 switch (hint) {
3235 case WRITE_LIFE_SHORT:
3236 return CURSEG_HOT_DATA;
3237 case WRITE_LIFE_EXTREME:
3238 return CURSEG_COLD_DATA;
3239 default:
3240 return CURSEG_WARM_DATA;
3241 }
3242}
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
3306 enum page_type type, enum temp_type temp)
3307{
3308 if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) {
3309 if (type == DATA) {
3310 if (temp == WARM)
3311 return WRITE_LIFE_NOT_SET;
3312 else if (temp == HOT)
3313 return WRITE_LIFE_SHORT;
3314 else if (temp == COLD)
3315 return WRITE_LIFE_EXTREME;
3316 } else {
3317 return WRITE_LIFE_NOT_SET;
3318 }
3319 } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) {
3320 if (type == DATA) {
3321 if (temp == WARM)
3322 return WRITE_LIFE_LONG;
3323 else if (temp == HOT)
3324 return WRITE_LIFE_SHORT;
3325 else if (temp == COLD)
3326 return WRITE_LIFE_EXTREME;
3327 } else if (type == NODE) {
3328 if (temp == WARM || temp == HOT)
3329 return WRITE_LIFE_NOT_SET;
3330 else if (temp == COLD)
3331 return WRITE_LIFE_NONE;
3332 } else if (type == META) {
3333 return WRITE_LIFE_MEDIUM;
3334 }
3335 }
3336 return WRITE_LIFE_NOT_SET;
3337}
3338
3339static int __get_segment_type_2(struct f2fs_io_info *fio)
3340{
3341 if (fio->type == DATA)
3342 return CURSEG_HOT_DATA;
3343 else
3344 return CURSEG_HOT_NODE;
3345}
3346
3347static int __get_segment_type_4(struct f2fs_io_info *fio)
3348{
3349 if (fio->type == DATA) {
3350 struct inode *inode = fio->page->mapping->host;
3351
3352 if (S_ISDIR(inode->i_mode))
3353 return CURSEG_HOT_DATA;
3354 else
3355 return CURSEG_COLD_DATA;
3356 } else {
3357 if (IS_DNODE(fio->page) && is_cold_node(fio->page))
3358 return CURSEG_WARM_NODE;
3359 else
3360 return CURSEG_COLD_NODE;
3361 }
3362}
3363
3364static int __get_segment_type_6(struct f2fs_io_info *fio)
3365{
3366 if (fio->type == DATA) {
3367 struct inode *inode = fio->page->mapping->host;
3368
3369 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
3370 return CURSEG_COLD_DATA_PINNED;
3371
3372 if (page_private_gcing(fio->page)) {
3373 if (fio->sbi->am.atgc_enabled &&
3374 (fio->io_type == FS_DATA_IO) &&
3375 (fio->sbi->gc_mode != GC_URGENT_HIGH))
3376 return CURSEG_ALL_DATA_ATGC;
3377 else
3378 return CURSEG_COLD_DATA;
3379 }
3380 if (file_is_cold(inode) || f2fs_need_compress_data(inode))
3381 return CURSEG_COLD_DATA;
3382 if (file_is_hot(inode) ||
3383 is_inode_flag_set(inode, FI_HOT_DATA) ||
3384 f2fs_is_atomic_file(inode) ||
3385 f2fs_is_volatile_file(inode))
3386 return CURSEG_HOT_DATA;
3387 return f2fs_rw_hint_to_seg_type(inode->i_write_hint);
3388 } else {
3389 if (IS_DNODE(fio->page))
3390 return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
3391 CURSEG_HOT_NODE;
3392 return CURSEG_COLD_NODE;
3393 }
3394}
3395
3396static int __get_segment_type(struct f2fs_io_info *fio)
3397{
3398 int type = 0;
3399
3400 switch (F2FS_OPTION(fio->sbi).active_logs) {
3401 case 2:
3402 type = __get_segment_type_2(fio);
3403 break;
3404 case 4:
3405 type = __get_segment_type_4(fio);
3406 break;
3407 case 6:
3408 type = __get_segment_type_6(fio);
3409 break;
3410 default:
3411 f2fs_bug_on(fio->sbi, true);
3412 }
3413
3414 if (IS_HOT(type))
3415 fio->temp = HOT;
3416 else if (IS_WARM(type))
3417 fio->temp = WARM;
3418 else
3419 fio->temp = COLD;
3420 return type;
3421}
3422
3423void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3424 block_t old_blkaddr, block_t *new_blkaddr,
3425 struct f2fs_summary *sum, int type,
3426 struct f2fs_io_info *fio)
3427{
3428 struct sit_info *sit_i = SIT_I(sbi);
3429 struct curseg_info *curseg = CURSEG_I(sbi, type);
3430 unsigned long long old_mtime;
3431 bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
3432 struct seg_entry *se = NULL;
3433
3434 down_read(&SM_I(sbi)->curseg_lock);
3435
3436 mutex_lock(&curseg->curseg_mutex);
3437 down_write(&sit_i->sentry_lock);
3438
3439 if (from_gc) {
3440 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
3441 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
3442 sanity_check_seg_type(sbi, se->type);
3443 f2fs_bug_on(sbi, IS_NODESEG(se->type));
3444 }
3445 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
3446
3447 f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);
3448
3449 f2fs_wait_discard_bio(sbi, *new_blkaddr);
3450
3451
3452
3453
3454
3455
3456 __add_sum_entry(sbi, type, sum);
3457
3458 __refresh_next_blkoff(sbi, curseg);
3459
3460 stat_inc_block_count(sbi, curseg);
3461
3462 if (from_gc) {
3463 old_mtime = get_segment_mtime(sbi, old_blkaddr);
3464 } else {
3465 update_segment_mtime(sbi, old_blkaddr, 0);
3466 old_mtime = 0;
3467 }
3468 update_segment_mtime(sbi, *new_blkaddr, old_mtime);
3469
3470
3471
3472
3473
3474 update_sit_entry(sbi, *new_blkaddr, 1);
3475 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
3476 update_sit_entry(sbi, old_blkaddr, -1);
3477
3478 if (!__has_curseg_space(sbi, curseg)) {
3479 if (from_gc)
3480 get_atssr_segment(sbi, type, se->type,
3481 AT_SSR, se->mtime);
3482 else
3483 sit_i->s_ops->allocate_segment(sbi, type, false);
3484 }
3485
3486
3487
3488
3489
3490 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3491 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
3492
3493 up_write(&sit_i->sentry_lock);
3494
3495 if (page && IS_NODESEG(type)) {
3496 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
3497
3498 f2fs_inode_chksum_set(sbi, page);
3499 }
3500
3501 if (fio) {
3502 struct f2fs_bio_info *io;
3503
3504 if (F2FS_IO_ALIGNED(sbi))
3505 fio->retry = false;
3506
3507 INIT_LIST_HEAD(&fio->list);
3508 fio->in_list = true;
3509 io = sbi->write_io[fio->type] + fio->temp;
3510 spin_lock(&io->io_lock);
3511 list_add_tail(&fio->list, &io->io_list);
3512 spin_unlock(&io->io_lock);
3513 }
3514
3515 mutex_unlock(&curseg->curseg_mutex);
3516
3517 up_read(&SM_I(sbi)->curseg_lock);
3518}
3519
3520void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
3521 block_t blkaddr, unsigned int blkcnt)
3522{
3523 if (!f2fs_is_multi_device(sbi))
3524 return;
3525
3526 while (1) {
3527 unsigned int devidx = f2fs_target_device_index(sbi, blkaddr);
3528 unsigned int blks = FDEV(devidx).end_blk - blkaddr + 1;
3529
3530
3531 f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO);
3532
3533
3534 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
3535 spin_lock(&sbi->dev_lock);
3536 f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
3537 spin_unlock(&sbi->dev_lock);
3538 }
3539
3540 if (blkcnt <= blks)
3541 break;
3542 blkcnt -= blks;
3543 blkaddr += blks;
3544 }
3545}
3546
3547static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
3548{
3549 int type = __get_segment_type(fio);
3550 bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);
3551
3552 if (keep_order)
3553 down_read(&fio->sbi->io_order_lock);
3554reallocate:
3555 f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
3556 &fio->new_blkaddr, sum, type, fio);
3557 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) {
3558 invalidate_mapping_pages(META_MAPPING(fio->sbi),
3559 fio->old_blkaddr, fio->old_blkaddr);
3560 f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr);
3561 }
3562
3563
3564 f2fs_submit_page_write(fio);
3565 if (fio->retry) {
3566 fio->old_blkaddr = fio->new_blkaddr;
3567 goto reallocate;
3568 }
3569
3570 f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1);
3571
3572 if (keep_order)
3573 up_read(&fio->sbi->io_order_lock);
3574}
3575
3576void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3577 enum iostat_type io_type)
3578{
3579 struct f2fs_io_info fio = {
3580 .sbi = sbi,
3581 .type = META,
3582 .temp = HOT,
3583 .op = REQ_OP_WRITE,
3584 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
3585 .old_blkaddr = page->index,
3586 .new_blkaddr = page->index,
3587 .page = page,
3588 .encrypted_page = NULL,
3589 .in_list = false,
3590 };
3591
3592 if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
3593 fio.op_flags &= ~REQ_META;
3594
3595 set_page_writeback(page);
3596 ClearPageError(page);
3597 f2fs_submit_page_write(&fio);
3598
3599 stat_inc_meta_count(sbi, page->index);
3600 f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);
3601}
3602
3603void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
3604{
3605 struct f2fs_summary sum;
3606
3607 set_summary(&sum, nid, 0, 0);
3608 do_write_page(&sum, fio);
3609
3610 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3611}
3612
3613void f2fs_outplace_write_data(struct dnode_of_data *dn,
3614 struct f2fs_io_info *fio)
3615{
3616 struct f2fs_sb_info *sbi = fio->sbi;
3617 struct f2fs_summary sum;
3618
3619 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
3620 set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
3621 do_write_page(&sum, fio);
3622 f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
3623
3624 f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);
3625}
3626
3627int f2fs_inplace_write_data(struct f2fs_io_info *fio)
3628{
3629 int err;
3630 struct f2fs_sb_info *sbi = fio->sbi;
3631 unsigned int segno;
3632
3633 fio->new_blkaddr = fio->old_blkaddr;
3634
3635 __get_segment_type(fio);
3636
3637 segno = GET_SEGNO(sbi, fio->new_blkaddr);
3638
3639 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
3640 set_sbi_flag(sbi, SBI_NEED_FSCK);
3641 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
3642 __func__, segno);
3643 err = -EFSCORRUPTED;
3644 goto drop_bio;
3645 }
3646
3647 if (f2fs_cp_error(sbi)) {
3648 err = -EIO;
3649 goto drop_bio;
3650 }
3651
3652 invalidate_mapping_pages(META_MAPPING(sbi),
3653 fio->new_blkaddr, fio->new_blkaddr);
3654
3655 stat_inc_inplace_blocks(fio->sbi);
3656
3657 if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE)))
3658 err = f2fs_merge_page_bio(fio);
3659 else
3660 err = f2fs_submit_page_bio(fio);
3661 if (!err) {
3662 f2fs_update_device_state(fio->sbi, fio->ino,
3663 fio->new_blkaddr, 1);
3664 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3665 }
3666
3667 return err;
3668drop_bio:
3669 if (fio->bio && *(fio->bio)) {
3670 struct bio *bio = *(fio->bio);
3671
3672 bio->bi_status = BLK_STS_IOERR;
3673 bio_endio(bio);
3674 *(fio->bio) = NULL;
3675 }
3676 return err;
3677}
3678
3679static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
3680 unsigned int segno)
3681{
3682 int i;
3683
3684 for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
3685 if (CURSEG_I(sbi, i)->segno == segno)
3686 break;
3687 }
3688 return i;
3689}
3690
3691void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3692 block_t old_blkaddr, block_t new_blkaddr,
3693 bool recover_curseg, bool recover_newaddr,
3694 bool from_gc)
3695{
3696 struct sit_info *sit_i = SIT_I(sbi);
3697 struct curseg_info *curseg;
3698 unsigned int segno, old_cursegno;
3699 struct seg_entry *se;
3700 int type;
3701 unsigned short old_blkoff;
3702 unsigned char old_alloc_type;
3703
3704 segno = GET_SEGNO(sbi, new_blkaddr);
3705 se = get_seg_entry(sbi, segno);
3706 type = se->type;
3707
3708 down_write(&SM_I(sbi)->curseg_lock);
3709
3710 if (!recover_curseg) {
3711
3712 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
3713 if (old_blkaddr == NULL_ADDR)
3714 type = CURSEG_COLD_DATA;
3715 else
3716 type = CURSEG_WARM_DATA;
3717 }
3718 } else {
3719 if (IS_CURSEG(sbi, segno)) {
3720
3721 type = __f2fs_get_curseg(sbi, segno);
3722 f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
3723 } else {
3724 type = CURSEG_WARM_DATA;
3725 }
3726 }
3727
3728 f2fs_bug_on(sbi, !IS_DATASEG(type));
3729 curseg = CURSEG_I(sbi, type);
3730
3731 mutex_lock(&curseg->curseg_mutex);
3732 down_write(&sit_i->sentry_lock);
3733
3734 old_cursegno = curseg->segno;
3735 old_blkoff = curseg->next_blkoff;
3736 old_alloc_type = curseg->alloc_type;
3737
3738
3739 if (segno != curseg->segno) {
3740 curseg->next_segno = segno;
3741 change_curseg(sbi, type, true);
3742 }
3743
3744 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
3745 __add_sum_entry(sbi, type, sum);
3746
3747 if (!recover_curseg || recover_newaddr) {
3748 if (!from_gc)
3749 update_segment_mtime(sbi, new_blkaddr, 0);
3750 update_sit_entry(sbi, new_blkaddr, 1);
3751 }
3752 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
3753 invalidate_mapping_pages(META_MAPPING(sbi),
3754 old_blkaddr, old_blkaddr);
3755 f2fs_invalidate_compress_page(sbi, old_blkaddr);
3756 if (!from_gc)
3757 update_segment_mtime(sbi, old_blkaddr, 0);
3758 update_sit_entry(sbi, old_blkaddr, -1);
3759 }
3760
3761 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3762 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
3763
3764 locate_dirty_segment(sbi, old_cursegno);
3765
3766 if (recover_curseg) {
3767 if (old_cursegno != curseg->segno) {
3768 curseg->next_segno = old_cursegno;
3769 change_curseg(sbi, type, true);
3770 }
3771 curseg->next_blkoff = old_blkoff;
3772 curseg->alloc_type = old_alloc_type;
3773 }
3774
3775 up_write(&sit_i->sentry_lock);
3776 mutex_unlock(&curseg->curseg_mutex);
3777 up_write(&SM_I(sbi)->curseg_lock);
3778}
3779
3780void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3781 block_t old_addr, block_t new_addr,
3782 unsigned char version, bool recover_curseg,
3783 bool recover_newaddr)
3784{
3785 struct f2fs_summary sum;
3786
3787 set_summary(&sum, dn->nid, dn->ofs_in_node, version);
3788
3789 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
3790 recover_curseg, recover_newaddr, false);
3791
3792 f2fs_update_data_blkaddr(dn, new_addr);
3793}
3794
3795void f2fs_wait_on_page_writeback(struct page *page,
3796 enum page_type type, bool ordered, bool locked)
3797{
3798 if (PageWriteback(page)) {
3799 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
3800
3801
3802 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
3803
3804 f2fs_submit_merged_ipu_write(sbi, NULL, page);
3805 if (ordered) {
3806 wait_on_page_writeback(page);
3807 f2fs_bug_on(sbi, locked && PageWriteback(page));
3808 } else {
3809 wait_for_stable_page(page);
3810 }
3811 }
3812}
3813
3814void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
3815{
3816 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3817 struct page *cpage;
3818
3819 if (!f2fs_post_read_required(inode))
3820 return;
3821
3822 if (!__is_valid_data_blkaddr(blkaddr))
3823 return;
3824
3825 cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
3826 if (cpage) {
3827 f2fs_wait_on_page_writeback(cpage, DATA, true, true);
3828 f2fs_put_page(cpage, 1);
3829 }
3830}
3831
3832void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3833 block_t len)
3834{
3835 block_t i;
3836
3837 for (i = 0; i < len; i++)
3838 f2fs_wait_on_block_writeback(inode, blkaddr + i);
3839}
3840
3841static int read_compacted_summaries(struct f2fs_sb_info *sbi)
3842{
3843 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3844 struct curseg_info *seg_i;
3845 unsigned char *kaddr;
3846 struct page *page;
3847 block_t start;
3848 int i, j, offset;
3849
3850 start = start_sum_block(sbi);
3851
3852 page = f2fs_get_meta_page(sbi, start++);
3853 if (IS_ERR(page))
3854 return PTR_ERR(page);
3855 kaddr = (unsigned char *)page_address(page);
3856
3857
3858 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3859 memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
3860
3861
3862 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3863 memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
3864 offset = 2 * SUM_JOURNAL_SIZE;
3865
3866
3867 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3868 unsigned short blk_off;
3869 unsigned int segno;
3870
3871 seg_i = CURSEG_I(sbi, i);
3872 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
3873 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
3874 seg_i->next_segno = segno;
3875 reset_curseg(sbi, i, 0);
3876 seg_i->alloc_type = ckpt->alloc_type[i];
3877 seg_i->next_blkoff = blk_off;
3878
3879 if (seg_i->alloc_type == SSR)
3880 blk_off = sbi->blocks_per_seg;
3881
3882 for (j = 0; j < blk_off; j++) {
3883 struct f2fs_summary *s;
3884
3885 s = (struct f2fs_summary *)(kaddr + offset);
3886 seg_i->sum_blk->entries[j] = *s;
3887 offset += SUMMARY_SIZE;
3888 if (offset + SUMMARY_SIZE <= PAGE_SIZE -
3889 SUM_FOOTER_SIZE)
3890 continue;
3891
3892 f2fs_put_page(page, 1);
3893 page = NULL;
3894
3895 page = f2fs_get_meta_page(sbi, start++);
3896 if (IS_ERR(page))
3897 return PTR_ERR(page);
3898 kaddr = (unsigned char *)page_address(page);
3899 offset = 0;
3900 }
3901 }
3902 f2fs_put_page(page, 1);
3903 return 0;
3904}
3905
3906static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
3907{
3908 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3909 struct f2fs_summary_block *sum;
3910 struct curseg_info *curseg;
3911 struct page *new;
3912 unsigned short blk_off;
3913 unsigned int segno = 0;
3914 block_t blk_addr = 0;
3915 int err = 0;
3916
3917
3918 if (IS_DATASEG(type)) {
3919 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
3920 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
3921 CURSEG_HOT_DATA]);
3922 if (__exist_node_summaries(sbi))
3923 blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
3924 else
3925 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
3926 } else {
3927 segno = le32_to_cpu(ckpt->cur_node_segno[type -
3928 CURSEG_HOT_NODE]);
3929 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
3930 CURSEG_HOT_NODE]);
3931 if (__exist_node_summaries(sbi))
3932 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
3933 type - CURSEG_HOT_NODE);
3934 else
3935 blk_addr = GET_SUM_BLOCK(sbi, segno);
3936 }
3937
3938 new = f2fs_get_meta_page(sbi, blk_addr);
3939 if (IS_ERR(new))
3940 return PTR_ERR(new);
3941 sum = (struct f2fs_summary_block *)page_address(new);
3942
3943 if (IS_NODESEG(type)) {
3944 if (__exist_node_summaries(sbi)) {
3945 struct f2fs_summary *ns = &sum->entries[0];
3946 int i;
3947
3948 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
3949 ns->version = 0;
3950 ns->ofs_in_node = 0;
3951 }
3952 } else {
3953 err = f2fs_restore_node_summary(sbi, segno, sum);
3954 if (err)
3955 goto out;
3956 }
3957 }
3958
3959
3960 curseg = CURSEG_I(sbi, type);
3961 mutex_lock(&curseg->curseg_mutex);
3962
3963
3964 down_write(&curseg->journal_rwsem);
3965 memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
3966 up_write(&curseg->journal_rwsem);
3967
3968 memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
3969 memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
3970 curseg->next_segno = segno;
3971 reset_curseg(sbi, type, 0);
3972 curseg->alloc_type = ckpt->alloc_type[type];
3973 curseg->next_blkoff = blk_off;
3974 mutex_unlock(&curseg->curseg_mutex);
3975out:
3976 f2fs_put_page(new, 1);
3977 return err;
3978}
3979
3980static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
3981{
3982 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
3983 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
3984 int type = CURSEG_HOT_DATA;
3985 int err;
3986
3987 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
3988 int npages = f2fs_npages_for_summary_flush(sbi, true);
3989
3990 if (npages >= 2)
3991 f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
3992 META_CP, true);
3993
3994
3995 err = read_compacted_summaries(sbi);
3996 if (err)
3997 return err;
3998 type = CURSEG_HOT_NODE;
3999 }
4000
4001 if (__exist_node_summaries(sbi))
4002 f2fs_ra_meta_pages(sbi,
4003 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
4004 NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
4005
4006 for (; type <= CURSEG_COLD_NODE; type++) {
4007 err = read_normal_summaries(sbi, type);
4008 if (err)
4009 return err;
4010 }
4011
4012
4013 if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
4014 sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
4015 f2fs_err(sbi, "invalid journal entries nats %u sits %u",
4016 nats_in_cursum(nat_j), sits_in_cursum(sit_j));
4017 return -EINVAL;
4018 }
4019
4020 return 0;
4021}
4022
4023static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
4024{
4025 struct page *page;
4026 unsigned char *kaddr;
4027 struct f2fs_summary *summary;
4028 struct curseg_info *seg_i;
4029 int written_size = 0;
4030 int i, j;
4031
4032 page = f2fs_grab_meta_page(sbi, blkaddr++);
4033 kaddr = (unsigned char *)page_address(page);
4034 memset(kaddr, 0, PAGE_SIZE);
4035
4036
4037 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
4038 memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
4039 written_size += SUM_JOURNAL_SIZE;
4040
4041
4042 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
4043 memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
4044 written_size += SUM_JOURNAL_SIZE;
4045
4046
4047 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
4048 unsigned short blkoff;
4049
4050 seg_i = CURSEG_I(sbi, i);
4051 if (sbi->ckpt->alloc_type[i] == SSR)
4052 blkoff = sbi->blocks_per_seg;
4053 else
4054 blkoff = curseg_blkoff(sbi, i);
4055
4056 for (j = 0; j < blkoff; j++) {
4057 if (!page) {
4058 page = f2fs_grab_meta_page(sbi, blkaddr++);
4059 kaddr = (unsigned char *)page_address(page);
4060 memset(kaddr, 0, PAGE_SIZE);
4061 written_size = 0;
4062 }
4063 summary = (struct f2fs_summary *)(kaddr + written_size);
4064 *summary = seg_i->sum_blk->entries[j];
4065 written_size += SUMMARY_SIZE;
4066
4067 if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
4068 SUM_FOOTER_SIZE)
4069 continue;
4070
4071 set_page_dirty(page);
4072 f2fs_put_page(page, 1);
4073 page = NULL;
4074 }
4075 }
4076 if (page) {
4077 set_page_dirty(page);
4078 f2fs_put_page(page, 1);
4079 }
4080}
4081
4082static void write_normal_summaries(struct f2fs_sb_info *sbi,
4083 block_t blkaddr, int type)
4084{
4085 int i, end;
4086
4087 if (IS_DATASEG(type))
4088 end = type + NR_CURSEG_DATA_TYPE;
4089 else
4090 end = type + NR_CURSEG_NODE_TYPE;
4091
4092 for (i = type; i < end; i++)
4093 write_current_sum_page(sbi, i, blkaddr + (i - type));
4094}
4095
4096void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4097{
4098 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
4099 write_compacted_summaries(sbi, start_blk);
4100 else
4101 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
4102}
4103
4104void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4105{
4106 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
4107}
4108
4109int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
4110 unsigned int val, int alloc)
4111{
4112 int i;
4113
4114 if (type == NAT_JOURNAL) {
4115 for (i = 0; i < nats_in_cursum(journal); i++) {
4116 if (le32_to_cpu(nid_in_journal(journal, i)) == val)
4117 return i;
4118 }
4119 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
4120 return update_nats_in_cursum(journal, 1);
4121 } else if (type == SIT_JOURNAL) {
4122 for (i = 0; i < sits_in_cursum(journal); i++)
4123 if (le32_to_cpu(segno_in_journal(journal, i)) == val)
4124 return i;
4125 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
4126 return update_sits_in_cursum(journal, 1);
4127 }
4128 return -1;
4129}
4130
4131static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
4132 unsigned int segno)
4133{
4134 return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));
4135}
4136
4137static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
4138 unsigned int start)
4139{
4140 struct sit_info *sit_i = SIT_I(sbi);
4141 struct page *page;
4142 pgoff_t src_off, dst_off;
4143
4144 src_off = current_sit_addr(sbi, start);
4145 dst_off = next_sit_addr(sbi, src_off);
4146
4147 page = f2fs_grab_meta_page(sbi, dst_off);
4148 seg_info_to_sit_page(sbi, page, start);
4149
4150 set_page_dirty(page);
4151 set_to_next_sit(sit_i, start);
4152
4153 return page;
4154}
4155
4156static struct sit_entry_set *grab_sit_entry_set(void)
4157{
4158 struct sit_entry_set *ses =
4159 f2fs_kmem_cache_alloc(sit_entry_set_slab,
4160 GFP_NOFS, true, NULL);
4161
4162 ses->entry_cnt = 0;
4163 INIT_LIST_HEAD(&ses->set_list);
4164 return ses;
4165}
4166
4167static void release_sit_entry_set(struct sit_entry_set *ses)
4168{
4169 list_del(&ses->set_list);
4170 kmem_cache_free(sit_entry_set_slab, ses);
4171}
4172
4173static void adjust_sit_entry_set(struct sit_entry_set *ses,
4174 struct list_head *head)
4175{
4176 struct sit_entry_set *next = ses;
4177
4178 if (list_is_last(&ses->set_list, head))
4179 return;
4180
4181 list_for_each_entry_continue(next, head, set_list)
4182 if (ses->entry_cnt <= next->entry_cnt)
4183 break;
4184
4185 list_move_tail(&ses->set_list, &next->set_list);
4186}
4187
4188static void add_sit_entry(unsigned int segno, struct list_head *head)
4189{
4190 struct sit_entry_set *ses;
4191 unsigned int start_segno = START_SEGNO(segno);
4192
4193 list_for_each_entry(ses, head, set_list) {
4194 if (ses->start_segno == start_segno) {
4195 ses->entry_cnt++;
4196 adjust_sit_entry_set(ses, head);
4197 return;
4198 }
4199 }
4200
4201 ses = grab_sit_entry_set();
4202
4203 ses->start_segno = start_segno;
4204 ses->entry_cnt++;
4205 list_add(&ses->set_list, head);
4206}
4207
4208static void add_sits_in_set(struct f2fs_sb_info *sbi)
4209{
4210 struct f2fs_sm_info *sm_info = SM_I(sbi);
4211 struct list_head *set_list = &sm_info->sit_entry_set;
4212 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
4213 unsigned int segno;
4214
4215 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
4216 add_sit_entry(segno, set_list);
4217}
4218
4219static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
4220{
4221 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4222 struct f2fs_journal *journal = curseg->journal;
4223 int i;
4224
4225 down_write(&curseg->journal_rwsem);
4226 for (i = 0; i < sits_in_cursum(journal); i++) {
4227 unsigned int segno;
4228 bool dirtied;
4229
4230 segno = le32_to_cpu(segno_in_journal(journal, i));
4231 dirtied = __mark_sit_entry_dirty(sbi, segno);
4232
4233 if (!dirtied)
4234 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
4235 }
4236 update_sits_in_cursum(journal, -i);
4237 up_write(&curseg->journal_rwsem);
4238}
4239
4240
4241
4242
4243
4244void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
4245{
4246 struct sit_info *sit_i = SIT_I(sbi);
4247 unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
4248 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4249 struct f2fs_journal *journal = curseg->journal;
4250 struct sit_entry_set *ses, *tmp;
4251 struct list_head *head = &SM_I(sbi)->sit_entry_set;
4252 bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
4253 struct seg_entry *se;
4254
4255 down_write(&sit_i->sentry_lock);
4256
4257 if (!sit_i->dirty_sentries)
4258 goto out;
4259
4260
4261
4262
4263
4264 add_sits_in_set(sbi);
4265
4266
4267
4268
4269
4270
4271 if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) ||
4272 !to_journal)
4273 remove_sits_in_journal(sbi);
4274
4275
4276
4277
4278
4279
4280 list_for_each_entry_safe(ses, tmp, head, set_list) {
4281 struct page *page = NULL;
4282 struct f2fs_sit_block *raw_sit = NULL;
4283 unsigned int start_segno = ses->start_segno;
4284 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
4285 (unsigned long)MAIN_SEGS(sbi));
4286 unsigned int segno = start_segno;
4287
4288 if (to_journal &&
4289 !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
4290 to_journal = false;
4291
4292 if (to_journal) {
4293 down_write(&curseg->journal_rwsem);
4294 } else {
4295 page = get_next_sit_page(sbi, start_segno);
4296 raw_sit = page_address(page);
4297 }
4298
4299
4300 for_each_set_bit_from(segno, bitmap, end) {
4301 int offset, sit_offset;
4302
4303 se = get_seg_entry(sbi, segno);
4304#ifdef CONFIG_F2FS_CHECK_FS
4305 if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
4306 SIT_VBLOCK_MAP_SIZE))
4307 f2fs_bug_on(sbi, 1);
4308#endif
4309
4310
4311 if (!(cpc->reason & CP_DISCARD)) {
4312 cpc->trim_start = segno;
4313 add_discard_addrs(sbi, cpc, false);
4314 }
4315
4316 if (to_journal) {
4317 offset = f2fs_lookup_journal_in_cursum(journal,
4318 SIT_JOURNAL, segno, 1);
4319 f2fs_bug_on(sbi, offset < 0);
4320 segno_in_journal(journal, offset) =
4321 cpu_to_le32(segno);
4322 seg_info_to_raw_sit(se,
4323 &sit_in_journal(journal, offset));
4324 check_block_count(sbi, segno,
4325 &sit_in_journal(journal, offset));
4326 } else {
4327 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
4328 seg_info_to_raw_sit(se,
4329 &raw_sit->entries[sit_offset]);
4330 check_block_count(sbi, segno,
4331 &raw_sit->entries[sit_offset]);
4332 }
4333
4334 __clear_bit(segno, bitmap);
4335 sit_i->dirty_sentries--;
4336 ses->entry_cnt--;
4337 }
4338
4339 if (to_journal)
4340 up_write(&curseg->journal_rwsem);
4341 else
4342 f2fs_put_page(page, 1);
4343
4344 f2fs_bug_on(sbi, ses->entry_cnt);
4345 release_sit_entry_set(ses);
4346 }
4347
4348 f2fs_bug_on(sbi, !list_empty(head));
4349 f2fs_bug_on(sbi, sit_i->dirty_sentries);
4350out:
4351 if (cpc->reason & CP_DISCARD) {
4352 __u64 trim_start = cpc->trim_start;
4353
4354 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
4355 add_discard_addrs(sbi, cpc, false);
4356
4357 cpc->trim_start = trim_start;
4358 }
4359 up_write(&sit_i->sentry_lock);
4360
4361 set_prefree_as_free_segments(sbi);
4362}
4363
4364static int build_sit_info(struct f2fs_sb_info *sbi)
4365{
4366 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4367 struct sit_info *sit_i;
4368 unsigned int sit_segs, start;
4369 char *src_bitmap, *bitmap;
4370 unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
4371 unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0;
4372
4373
4374 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
4375 if (!sit_i)
4376 return -ENOMEM;
4377
4378 SM_I(sbi)->sit_info = sit_i;
4379
4380 sit_i->sentries =
4381 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
4382 MAIN_SEGS(sbi)),
4383 GFP_KERNEL);
4384 if (!sit_i->sentries)
4385 return -ENOMEM;
4386
4387 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4388 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
4389 GFP_KERNEL);
4390 if (!sit_i->dirty_sentries_bitmap)
4391 return -ENOMEM;
4392
4393#ifdef CONFIG_F2FS_CHECK_FS
4394 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map);
4395#else
4396 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map);
4397#endif
4398 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4399 if (!sit_i->bitmap)
4400 return -ENOMEM;
4401
4402 bitmap = sit_i->bitmap;
4403
4404 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4405 sit_i->sentries[start].cur_valid_map = bitmap;
4406 bitmap += SIT_VBLOCK_MAP_SIZE;
4407
4408 sit_i->sentries[start].ckpt_valid_map = bitmap;
4409 bitmap += SIT_VBLOCK_MAP_SIZE;
4410
4411#ifdef CONFIG_F2FS_CHECK_FS
4412 sit_i->sentries[start].cur_valid_map_mir = bitmap;
4413 bitmap += SIT_VBLOCK_MAP_SIZE;
4414#endif
4415
4416 if (discard_map) {
4417 sit_i->sentries[start].discard_map = bitmap;
4418 bitmap += SIT_VBLOCK_MAP_SIZE;
4419 }
4420 }
4421
4422 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4423 if (!sit_i->tmp_map)
4424 return -ENOMEM;
4425
4426 if (__is_large_section(sbi)) {
4427 sit_i->sec_entries =
4428 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
4429 MAIN_SECS(sbi)),
4430 GFP_KERNEL);
4431 if (!sit_i->sec_entries)
4432 return -ENOMEM;
4433 }
4434
4435
4436 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
4437
4438
4439 sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
4440 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
4441
4442 sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
4443 if (!sit_i->sit_bitmap)
4444 return -ENOMEM;
4445
4446#ifdef CONFIG_F2FS_CHECK_FS
4447 sit_i->sit_bitmap_mir = kmemdup(src_bitmap,
4448 sit_bitmap_size, GFP_KERNEL);
4449 if (!sit_i->sit_bitmap_mir)
4450 return -ENOMEM;
4451
4452 sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
4453 main_bitmap_size, GFP_KERNEL);
4454 if (!sit_i->invalid_segmap)
4455 return -ENOMEM;
4456#endif
4457
4458
4459 sit_i->s_ops = &default_salloc_ops;
4460
4461 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
4462 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
4463 sit_i->written_valid_blocks = 0;
4464 sit_i->bitmap_size = sit_bitmap_size;
4465 sit_i->dirty_sentries = 0;
4466 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
4467 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
4468 sit_i->mounted_time = ktime_get_boottime_seconds();
4469 init_rwsem(&sit_i->sentry_lock);
4470 return 0;
4471}
4472
4473static int build_free_segmap(struct f2fs_sb_info *sbi)
4474{
4475 struct free_segmap_info *free_i;
4476 unsigned int bitmap_size, sec_bitmap_size;
4477
4478
4479 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
4480 if (!free_i)
4481 return -ENOMEM;
4482
4483 SM_I(sbi)->free_info = free_i;
4484
4485 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4486 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
4487 if (!free_i->free_segmap)
4488 return -ENOMEM;
4489
4490 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4491 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
4492 if (!free_i->free_secmap)
4493 return -ENOMEM;
4494
4495
4496 memset(free_i->free_segmap, 0xff, bitmap_size);
4497 memset(free_i->free_secmap, 0xff, sec_bitmap_size);
4498
4499
4500 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
4501 free_i->free_segments = 0;
4502 free_i->free_sections = 0;
4503 spin_lock_init(&free_i->segmap_lock);
4504 return 0;
4505}
4506
4507static int build_curseg(struct f2fs_sb_info *sbi)
4508{
4509 struct curseg_info *array;
4510 int i;
4511
4512 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
4513 sizeof(*array)), GFP_KERNEL);
4514 if (!array)
4515 return -ENOMEM;
4516
4517 SM_I(sbi)->curseg_array = array;
4518
4519 for (i = 0; i < NO_CHECK_TYPE; i++) {
4520 mutex_init(&array[i].curseg_mutex);
4521 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
4522 if (!array[i].sum_blk)
4523 return -ENOMEM;
4524 init_rwsem(&array[i].journal_rwsem);
4525 array[i].journal = f2fs_kzalloc(sbi,
4526 sizeof(struct f2fs_journal), GFP_KERNEL);
4527 if (!array[i].journal)
4528 return -ENOMEM;
4529 if (i < NR_PERSISTENT_LOG)
4530 array[i].seg_type = CURSEG_HOT_DATA + i;
4531 else if (i == CURSEG_COLD_DATA_PINNED)
4532 array[i].seg_type = CURSEG_COLD_DATA;
4533 else if (i == CURSEG_ALL_DATA_ATGC)
4534 array[i].seg_type = CURSEG_COLD_DATA;
4535 array[i].segno = NULL_SEGNO;
4536 array[i].next_blkoff = 0;
4537 array[i].inited = false;
4538 }
4539 return restore_curseg_summaries(sbi);
4540}
4541
4542static int build_sit_entries(struct f2fs_sb_info *sbi)
4543{
4544 struct sit_info *sit_i = SIT_I(sbi);
4545 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4546 struct f2fs_journal *journal = curseg->journal;
4547 struct seg_entry *se;
4548 struct f2fs_sit_entry sit;
4549 int sit_blk_cnt = SIT_BLK_CNT(sbi);
4550 unsigned int i, start, end;
4551 unsigned int readed, start_blk = 0;
4552 int err = 0;
4553 block_t total_node_blocks = 0;
4554
4555 do {
4556 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS,
4557 META_SIT, true);
4558
4559 start = start_blk * sit_i->sents_per_block;
4560 end = (start_blk + readed) * sit_i->sents_per_block;
4561
4562 for (; start < end && start < MAIN_SEGS(sbi); start++) {
4563 struct f2fs_sit_block *sit_blk;
4564 struct page *page;
4565
4566 se = &sit_i->sentries[start];
4567 page = get_current_sit_page(sbi, start);
4568 if (IS_ERR(page))
4569 return PTR_ERR(page);
4570 sit_blk = (struct f2fs_sit_block *)page_address(page);
4571 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
4572 f2fs_put_page(page, 1);
4573
4574 err = check_block_count(sbi, start, &sit);
4575 if (err)
4576 return err;
4577 seg_info_from_raw_sit(se, &sit);
4578 if (IS_NODESEG(se->type))
4579 total_node_blocks += se->valid_blocks;
4580
4581 if (f2fs_block_unit_discard(sbi)) {
4582
4583 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4584 memset(se->discard_map, 0xff,
4585 SIT_VBLOCK_MAP_SIZE);
4586 } else {
4587 memcpy(se->discard_map,
4588 se->cur_valid_map,
4589 SIT_VBLOCK_MAP_SIZE);
4590 sbi->discard_blks +=
4591 sbi->blocks_per_seg -
4592 se->valid_blocks;
4593 }
4594 }
4595
4596 if (__is_large_section(sbi))
4597 get_sec_entry(sbi, start)->valid_blocks +=
4598 se->valid_blocks;
4599 }
4600 start_blk += readed;
4601 } while (start_blk < sit_blk_cnt);
4602
4603 down_read(&curseg->journal_rwsem);
4604 for (i = 0; i < sits_in_cursum(journal); i++) {
4605 unsigned int old_valid_blocks;
4606
4607 start = le32_to_cpu(segno_in_journal(journal, i));
4608 if (start >= MAIN_SEGS(sbi)) {
4609 f2fs_err(sbi, "Wrong journal entry on segno %u",
4610 start);
4611 err = -EFSCORRUPTED;
4612 break;
4613 }
4614
4615 se = &sit_i->sentries[start];
4616 sit = sit_in_journal(journal, i);
4617
4618 old_valid_blocks = se->valid_blocks;
4619 if (IS_NODESEG(se->type))
4620 total_node_blocks -= old_valid_blocks;
4621
4622 err = check_block_count(sbi, start, &sit);
4623 if (err)
4624 break;
4625 seg_info_from_raw_sit(se, &sit);
4626 if (IS_NODESEG(se->type))
4627 total_node_blocks += se->valid_blocks;
4628
4629 if (f2fs_block_unit_discard(sbi)) {
4630 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4631 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
4632 } else {
4633 memcpy(se->discard_map, se->cur_valid_map,
4634 SIT_VBLOCK_MAP_SIZE);
4635 sbi->discard_blks += old_valid_blocks;
4636 sbi->discard_blks -= se->valid_blocks;
4637 }
4638 }
4639
4640 if (__is_large_section(sbi)) {
4641 get_sec_entry(sbi, start)->valid_blocks +=
4642 se->valid_blocks;
4643 get_sec_entry(sbi, start)->valid_blocks -=
4644 old_valid_blocks;
4645 }
4646 }
4647 up_read(&curseg->journal_rwsem);
4648
4649 if (!err && total_node_blocks != valid_node_count(sbi)) {
4650 f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
4651 total_node_blocks, valid_node_count(sbi));
4652 err = -EFSCORRUPTED;
4653 }
4654
4655 return err;
4656}
4657
4658static void init_free_segmap(struct f2fs_sb_info *sbi)
4659{
4660 unsigned int start;
4661 int type;
4662 struct seg_entry *sentry;
4663
4664 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4665 if (f2fs_usable_blks_in_seg(sbi, start) == 0)
4666 continue;
4667 sentry = get_seg_entry(sbi, start);
4668 if (!sentry->valid_blocks)
4669 __set_free(sbi, start);
4670 else
4671 SIT_I(sbi)->written_valid_blocks +=
4672 sentry->valid_blocks;
4673 }
4674
4675
4676 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
4677 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
4678
4679 __set_test_and_inuse(sbi, curseg_t->segno);
4680 }
4681}
4682
4683static void init_dirty_segmap(struct f2fs_sb_info *sbi)
4684{
4685 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4686 struct free_segmap_info *free_i = FREE_I(sbi);
4687 unsigned int segno = 0, offset = 0, secno;
4688 block_t valid_blocks, usable_blks_in_seg;
4689 block_t blks_per_sec = BLKS_PER_SEC(sbi);
4690
4691 while (1) {
4692
4693 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
4694 if (segno >= MAIN_SEGS(sbi))
4695 break;
4696 offset = segno + 1;
4697 valid_blocks = get_valid_blocks(sbi, segno, false);
4698 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
4699 if (valid_blocks == usable_blks_in_seg || !valid_blocks)
4700 continue;
4701 if (valid_blocks > usable_blks_in_seg) {
4702 f2fs_bug_on(sbi, 1);
4703 continue;
4704 }
4705 mutex_lock(&dirty_i->seglist_lock);
4706 __locate_dirty_segment(sbi, segno, DIRTY);
4707 mutex_unlock(&dirty_i->seglist_lock);
4708 }
4709
4710 if (!__is_large_section(sbi))
4711 return;
4712
4713 mutex_lock(&dirty_i->seglist_lock);
4714 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
4715 valid_blocks = get_valid_blocks(sbi, segno, true);
4716 secno = GET_SEC_FROM_SEG(sbi, segno);
4717
4718 if (!valid_blocks || valid_blocks == blks_per_sec)
4719 continue;
4720 if (IS_CURSEC(sbi, secno))
4721 continue;
4722 set_bit(secno, dirty_i->dirty_secmap);
4723 }
4724 mutex_unlock(&dirty_i->seglist_lock);
4725}
4726
4727static int init_victim_secmap(struct f2fs_sb_info *sbi)
4728{
4729 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4730 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4731
4732 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4733 if (!dirty_i->victim_secmap)
4734 return -ENOMEM;
4735 return 0;
4736}
4737
4738static int build_dirty_segmap(struct f2fs_sb_info *sbi)
4739{
4740 struct dirty_seglist_info *dirty_i;
4741 unsigned int bitmap_size, i;
4742
4743
4744 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
4745 GFP_KERNEL);
4746 if (!dirty_i)
4747 return -ENOMEM;
4748
4749 SM_I(sbi)->dirty_info = dirty_i;
4750 mutex_init(&dirty_i->seglist_lock);
4751
4752 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4753
4754 for (i = 0; i < NR_DIRTY_TYPE; i++) {
4755 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
4756 GFP_KERNEL);
4757 if (!dirty_i->dirty_segmap[i])
4758 return -ENOMEM;
4759 }
4760
4761 if (__is_large_section(sbi)) {
4762 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4763 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
4764 bitmap_size, GFP_KERNEL);
4765 if (!dirty_i->dirty_secmap)
4766 return -ENOMEM;
4767 }
4768
4769 init_dirty_segmap(sbi);
4770 return init_victim_secmap(sbi);
4771}
4772
4773static int sanity_check_curseg(struct f2fs_sb_info *sbi)
4774{
4775 int i;
4776
4777
4778
4779
4780
4781 for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4782 struct curseg_info *curseg = CURSEG_I(sbi, i);
4783 struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
4784 unsigned int blkofs = curseg->next_blkoff;
4785
4786 if (f2fs_sb_has_readonly(sbi) &&
4787 i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE)
4788 continue;
4789
4790 sanity_check_seg_type(sbi, curseg->seg_type);
4791
4792 if (f2fs_test_bit(blkofs, se->cur_valid_map))
4793 goto out;
4794
4795 if (curseg->alloc_type == SSR)
4796 continue;
4797
4798 for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {
4799 if (!f2fs_test_bit(blkofs, se->cur_valid_map))
4800 continue;
4801out:
4802 f2fs_err(sbi,
4803 "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
4804 i, curseg->segno, curseg->alloc_type,
4805 curseg->next_blkoff, blkofs);
4806 return -EFSCORRUPTED;
4807 }
4808 }
4809 return 0;
4810}
4811
4812#ifdef CONFIG_BLK_DEV_ZONED
4813
4814static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
4815 struct f2fs_dev_info *fdev,
4816 struct blk_zone *zone)
4817{
4818 unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno;
4819 block_t zone_block, wp_block, last_valid_block;
4820 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4821 int i, s, b, ret;
4822 struct seg_entry *se;
4823
4824 if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4825 return 0;
4826
4827 wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block);
4828 wp_segno = GET_SEGNO(sbi, wp_block);
4829 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4830 zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
4831 zone_segno = GET_SEGNO(sbi, zone_block);
4832 zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);
4833
4834 if (zone_segno >= MAIN_SEGS(sbi))
4835 return 0;
4836
4837
4838
4839
4840
4841 for (i = 0; i < NO_CHECK_TYPE; i++)
4842 if (zone_secno == GET_SEC_FROM_SEG(sbi,
4843 CURSEG_I(sbi, i)->segno))
4844 return 0;
4845
4846
4847
4848
4849 last_valid_block = zone_block - 1;
4850 for (s = sbi->segs_per_sec - 1; s >= 0; s--) {
4851 segno = zone_segno + s;
4852 se = get_seg_entry(sbi, segno);
4853 for (b = sbi->blocks_per_seg - 1; b >= 0; b--)
4854 if (f2fs_test_bit(b, se->cur_valid_map)) {
4855 last_valid_block = START_BLOCK(sbi, segno) + b;
4856 break;
4857 }
4858 if (last_valid_block >= zone_block)
4859 break;
4860 }
4861
4862
4863
4864
4865
4866
4867
4868 if (last_valid_block >= wp_block) {
4869 f2fs_notice(sbi, "Valid block beyond write pointer: "
4870 "valid block[0x%x,0x%x] wp[0x%x,0x%x]",
4871 GET_SEGNO(sbi, last_valid_block),
4872 GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),
4873 wp_segno, wp_blkoff);
4874 return 0;
4875 }
4876
4877
4878
4879
4880
4881 if (last_valid_block + 1 == zone_block && zone->wp != zone->start) {
4882 f2fs_notice(sbi,
4883 "Zone without valid block has non-zero write "
4884 "pointer. Reset the write pointer: wp[0x%x,0x%x]",
4885 wp_segno, wp_blkoff);
4886 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
4887 zone->len >> log_sectors_per_block);
4888 if (ret) {
4889 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4890 fdev->path, ret);
4891 return ret;
4892 }
4893 }
4894
4895 return 0;
4896}
4897
4898static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
4899 block_t zone_blkaddr)
4900{
4901 int i;
4902
4903 for (i = 0; i < sbi->s_ndevs; i++) {
4904 if (!bdev_is_zoned(FDEV(i).bdev))
4905 continue;
4906 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&
4907 zone_blkaddr <= FDEV(i).end_blk))
4908 return &FDEV(i);
4909 }
4910
4911 return NULL;
4912}
4913
4914static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
4915 void *data)
4916{
4917 memcpy(data, zone, sizeof(struct blk_zone));
4918 return 0;
4919}
4920
4921static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
4922{
4923 struct curseg_info *cs = CURSEG_I(sbi, type);
4924 struct f2fs_dev_info *zbd;
4925 struct blk_zone zone;
4926 unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
4927 block_t cs_zone_block, wp_block;
4928 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4929 sector_t zone_sector;
4930 int err;
4931
4932 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4933 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4934
4935 zbd = get_target_zoned_dev(sbi, cs_zone_block);
4936 if (!zbd)
4937 return 0;
4938
4939
4940 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4941 << log_sectors_per_block;
4942 err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4943 report_one_zone_cb, &zone);
4944 if (err != 1) {
4945 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4946 zbd->path, err);
4947 return err;
4948 }
4949
4950 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4951 return 0;
4952
4953 wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
4954 wp_segno = GET_SEGNO(sbi, wp_block);
4955 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4956 wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
4957
4958 if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
4959 wp_sector_off == 0)
4960 return 0;
4961
4962 f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
4963 "curseg[0x%x,0x%x] wp[0x%x,0x%x]",
4964 type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff);
4965
4966 f2fs_notice(sbi, "Assign new section to curseg[%d]: "
4967 "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff);
4968
4969 f2fs_allocate_new_section(sbi, type, true);
4970
4971
4972 if (check_zone_write_pointer(sbi, zbd, &zone))
4973 return -EIO;
4974
4975
4976 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4977 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4978
4979 zbd = get_target_zoned_dev(sbi, cs_zone_block);
4980 if (!zbd)
4981 return 0;
4982
4983 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4984 << log_sectors_per_block;
4985 err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4986 report_one_zone_cb, &zone);
4987 if (err != 1) {
4988 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4989 zbd->path, err);
4990 return err;
4991 }
4992
4993 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4994 return 0;
4995
4996 if (zone.wp != zone.start) {
4997 f2fs_notice(sbi,
4998 "New zone for curseg[%d] is not yet discarded. "
4999 "Reset the zone: curseg[0x%x,0x%x]",
5000 type, cs->segno, cs->next_blkoff);
5001 err = __f2fs_issue_discard_zone(sbi, zbd->bdev,
5002 zone_sector >> log_sectors_per_block,
5003 zone.len >> log_sectors_per_block);
5004 if (err) {
5005 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
5006 zbd->path, err);
5007 return err;
5008 }
5009 }
5010
5011 return 0;
5012}
5013
5014int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
5015{
5016 int i, ret;
5017
5018 for (i = 0; i < NR_PERSISTENT_LOG; i++) {
5019 ret = fix_curseg_write_pointer(sbi, i);
5020 if (ret)
5021 return ret;
5022 }
5023
5024 return 0;
5025}
5026
5027struct check_zone_write_pointer_args {
5028 struct f2fs_sb_info *sbi;
5029 struct f2fs_dev_info *fdev;
5030};
5031
5032static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
5033 void *data)
5034{
5035 struct check_zone_write_pointer_args *args;
5036
5037 args = (struct check_zone_write_pointer_args *)data;
5038
5039 return check_zone_write_pointer(args->sbi, args->fdev, zone);
5040}
5041
5042int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
5043{
5044 int i, ret;
5045 struct check_zone_write_pointer_args args;
5046
5047 for (i = 0; i < sbi->s_ndevs; i++) {
5048 if (!bdev_is_zoned(FDEV(i).bdev))
5049 continue;
5050
5051 args.sbi = sbi;
5052 args.fdev = &FDEV(i);
5053 ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
5054 check_zone_write_pointer_cb, &args);
5055 if (ret < 0)
5056 return ret;
5057 }
5058
5059 return 0;
5060}
5061
5062static bool is_conv_zone(struct f2fs_sb_info *sbi, unsigned int zone_idx,
5063 unsigned int dev_idx)
5064{
5065 if (!bdev_is_zoned(FDEV(dev_idx).bdev))
5066 return true;
5067 return !test_bit(zone_idx, FDEV(dev_idx).blkz_seq);
5068}
5069
5070
5071static unsigned int get_zone_idx(struct f2fs_sb_info *sbi, unsigned int secno,
5072 int dev_idx)
5073{
5074 block_t sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
5075
5076 return (sec_start_blkaddr - FDEV(dev_idx).start_blk) >>
5077 sbi->log_blocks_per_blkz;
5078}
5079
5080
5081
5082
5083
5084static inline unsigned int f2fs_usable_zone_segs_in_sec(
5085 struct f2fs_sb_info *sbi, unsigned int segno)
5086{
5087 unsigned int dev_idx, zone_idx, unusable_segs_in_sec;
5088
5089 dev_idx = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno));
5090 zone_idx = get_zone_idx(sbi, GET_SEC_FROM_SEG(sbi, segno), dev_idx);
5091
5092
5093 if (is_conv_zone(sbi, zone_idx, dev_idx))
5094 return sbi->segs_per_sec;
5095
5096
5097
5098
5099
5100 if (!FDEV(dev_idx).zone_capacity_blocks)
5101 return sbi->segs_per_sec;
5102
5103
5104 unusable_segs_in_sec = (sbi->blocks_per_blkz -
5105 FDEV(dev_idx).zone_capacity_blocks[zone_idx]) >>
5106 sbi->log_blocks_per_seg;
5107 return sbi->segs_per_sec - unusable_segs_in_sec;
5108}
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118static inline unsigned int f2fs_usable_zone_blks_in_seg(
5119 struct f2fs_sb_info *sbi, unsigned int segno)
5120{
5121 block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
5122 unsigned int zone_idx, dev_idx, secno;
5123
5124 secno = GET_SEC_FROM_SEG(sbi, segno);
5125 seg_start = START_BLOCK(sbi, segno);
5126 dev_idx = f2fs_target_device_index(sbi, seg_start);
5127 zone_idx = get_zone_idx(sbi, secno, dev_idx);
5128
5129
5130
5131
5132
5133 if (is_conv_zone(sbi, zone_idx, dev_idx))
5134 return sbi->blocks_per_seg;
5135
5136 if (!FDEV(dev_idx).zone_capacity_blocks)
5137 return sbi->blocks_per_seg;
5138
5139 sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
5140 sec_cap_blkaddr = sec_start_blkaddr +
5141 FDEV(dev_idx).zone_capacity_blocks[zone_idx];
5142
5143
5144
5145
5146
5147
5148
5149 if (seg_start >= sec_cap_blkaddr)
5150 return 0;
5151 if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)
5152 return sec_cap_blkaddr - seg_start;
5153
5154 return sbi->blocks_per_seg;
5155}
5156#else
5157int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
5158{
5159 return 0;
5160}
5161
5162int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
5163{
5164 return 0;
5165}
5166
5167static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
5168 unsigned int segno)
5169{
5170 return 0;
5171}
5172
5173static inline unsigned int f2fs_usable_zone_segs_in_sec(struct f2fs_sb_info *sbi,
5174 unsigned int segno)
5175{
5176 return 0;
5177}
5178#endif
5179unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
5180 unsigned int segno)
5181{
5182 if (f2fs_sb_has_blkzoned(sbi))
5183 return f2fs_usable_zone_blks_in_seg(sbi, segno);
5184
5185 return sbi->blocks_per_seg;
5186}
5187
5188unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
5189 unsigned int segno)
5190{
5191 if (f2fs_sb_has_blkzoned(sbi))
5192 return f2fs_usable_zone_segs_in_sec(sbi, segno);
5193
5194 return sbi->segs_per_sec;
5195}
5196
5197
5198
5199
5200static void init_min_max_mtime(struct f2fs_sb_info *sbi)
5201{
5202 struct sit_info *sit_i = SIT_I(sbi);
5203 unsigned int segno;
5204
5205 down_write(&sit_i->sentry_lock);
5206
5207 sit_i->min_mtime = ULLONG_MAX;
5208
5209 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
5210 unsigned int i;
5211 unsigned long long mtime = 0;
5212
5213 for (i = 0; i < sbi->segs_per_sec; i++)
5214 mtime += get_seg_entry(sbi, segno + i)->mtime;
5215
5216 mtime = div_u64(mtime, sbi->segs_per_sec);
5217
5218 if (sit_i->min_mtime > mtime)
5219 sit_i->min_mtime = mtime;
5220 }
5221 sit_i->max_mtime = get_mtime(sbi, false);
5222 sit_i->dirty_max_mtime = 0;
5223 up_write(&sit_i->sentry_lock);
5224}
5225
5226int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
5227{
5228 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
5229 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
5230 struct f2fs_sm_info *sm_info;
5231 int err;
5232
5233 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
5234 if (!sm_info)
5235 return -ENOMEM;
5236
5237
5238 sbi->sm_info = sm_info;
5239 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
5240 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
5241 sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
5242 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
5243 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
5244 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
5245 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
5246 sm_info->rec_prefree_segments = sm_info->main_segments *
5247 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
5248 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
5249 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
5250
5251 if (!f2fs_lfs_mode(sbi))
5252 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
5253 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
5254 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
5255 sm_info->min_seq_blocks = sbi->blocks_per_seg;
5256 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
5257 sm_info->min_ssr_sections = reserved_sections(sbi);
5258
5259 INIT_LIST_HEAD(&sm_info->sit_entry_set);
5260
5261 init_rwsem(&sm_info->curseg_lock);
5262
5263 if (!f2fs_readonly(sbi->sb)) {
5264 err = f2fs_create_flush_cmd_control(sbi);
5265 if (err)
5266 return err;
5267 }
5268
5269 err = create_discard_cmd_control(sbi);
5270 if (err)
5271 return err;
5272
5273 err = build_sit_info(sbi);
5274 if (err)
5275 return err;
5276 err = build_free_segmap(sbi);
5277 if (err)
5278 return err;
5279 err = build_curseg(sbi);
5280 if (err)
5281 return err;
5282
5283
5284 err = build_sit_entries(sbi);
5285 if (err)
5286 return err;
5287
5288 init_free_segmap(sbi);
5289 err = build_dirty_segmap(sbi);
5290 if (err)
5291 return err;
5292
5293 err = sanity_check_curseg(sbi);
5294 if (err)
5295 return err;
5296
5297 init_min_max_mtime(sbi);
5298 return 0;
5299}
5300
5301static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
5302 enum dirty_type dirty_type)
5303{
5304 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5305
5306 mutex_lock(&dirty_i->seglist_lock);
5307 kvfree(dirty_i->dirty_segmap[dirty_type]);
5308 dirty_i->nr_dirty[dirty_type] = 0;
5309 mutex_unlock(&dirty_i->seglist_lock);
5310}
5311
5312static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
5313{
5314 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5315
5316 kvfree(dirty_i->victim_secmap);
5317}
5318
5319static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
5320{
5321 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5322 int i;
5323
5324 if (!dirty_i)
5325 return;
5326
5327
5328 for (i = 0; i < NR_DIRTY_TYPE; i++)
5329 discard_dirty_segmap(sbi, i);
5330
5331 if (__is_large_section(sbi)) {
5332 mutex_lock(&dirty_i->seglist_lock);
5333 kvfree(dirty_i->dirty_secmap);
5334 mutex_unlock(&dirty_i->seglist_lock);
5335 }
5336
5337 destroy_victim_secmap(sbi);
5338 SM_I(sbi)->dirty_info = NULL;
5339 kfree(dirty_i);
5340}
5341
5342static void destroy_curseg(struct f2fs_sb_info *sbi)
5343{
5344 struct curseg_info *array = SM_I(sbi)->curseg_array;
5345 int i;
5346
5347 if (!array)
5348 return;
5349 SM_I(sbi)->curseg_array = NULL;
5350 for (i = 0; i < NR_CURSEG_TYPE; i++) {
5351 kfree(array[i].sum_blk);
5352 kfree(array[i].journal);
5353 }
5354 kfree(array);
5355}
5356
5357static void destroy_free_segmap(struct f2fs_sb_info *sbi)
5358{
5359 struct free_segmap_info *free_i = SM_I(sbi)->free_info;
5360
5361 if (!free_i)
5362 return;
5363 SM_I(sbi)->free_info = NULL;
5364 kvfree(free_i->free_segmap);
5365 kvfree(free_i->free_secmap);
5366 kfree(free_i);
5367}
5368
5369static void destroy_sit_info(struct f2fs_sb_info *sbi)
5370{
5371 struct sit_info *sit_i = SIT_I(sbi);
5372
5373 if (!sit_i)
5374 return;
5375
5376 if (sit_i->sentries)
5377 kvfree(sit_i->bitmap);
5378 kfree(sit_i->tmp_map);
5379
5380 kvfree(sit_i->sentries);
5381 kvfree(sit_i->sec_entries);
5382 kvfree(sit_i->dirty_sentries_bitmap);
5383
5384 SM_I(sbi)->sit_info = NULL;
5385 kvfree(sit_i->sit_bitmap);
5386#ifdef CONFIG_F2FS_CHECK_FS
5387 kvfree(sit_i->sit_bitmap_mir);
5388 kvfree(sit_i->invalid_segmap);
5389#endif
5390 kfree(sit_i);
5391}
5392
5393void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
5394{
5395 struct f2fs_sm_info *sm_info = SM_I(sbi);
5396
5397 if (!sm_info)
5398 return;
5399 f2fs_destroy_flush_cmd_control(sbi, true);
5400 destroy_discard_cmd_control(sbi);
5401 destroy_dirty_segmap(sbi);
5402 destroy_curseg(sbi);
5403 destroy_free_segmap(sbi);
5404 destroy_sit_info(sbi);
5405 sbi->sm_info = NULL;
5406 kfree(sm_info);
5407}
5408
5409int __init f2fs_create_segment_manager_caches(void)
5410{
5411 discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry",
5412 sizeof(struct discard_entry));
5413 if (!discard_entry_slab)
5414 goto fail;
5415
5416 discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd",
5417 sizeof(struct discard_cmd));
5418 if (!discard_cmd_slab)
5419 goto destroy_discard_entry;
5420
5421 sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set",
5422 sizeof(struct sit_entry_set));
5423 if (!sit_entry_set_slab)
5424 goto destroy_discard_cmd;
5425
5426 inmem_entry_slab = f2fs_kmem_cache_create("f2fs_inmem_page_entry",
5427 sizeof(struct inmem_pages));
5428 if (!inmem_entry_slab)
5429 goto destroy_sit_entry_set;
5430 return 0;
5431
5432destroy_sit_entry_set:
5433 kmem_cache_destroy(sit_entry_set_slab);
5434destroy_discard_cmd:
5435 kmem_cache_destroy(discard_cmd_slab);
5436destroy_discard_entry:
5437 kmem_cache_destroy(discard_entry_slab);
5438fail:
5439 return -ENOMEM;
5440}
5441
5442void f2fs_destroy_segment_manager_caches(void)
5443{
5444 kmem_cache_destroy(sit_entry_set_slab);
5445 kmem_cache_destroy(discard_cmd_slab);
5446 kmem_cache_destroy(discard_entry_slab);
5447 kmem_cache_destroy(inmem_entry_slab);
5448}
5449
