1
2
3
4
5
6
7
8#include <linux/fs.h>
9#include <linux/f2fs_fs.h>
10#include <linux/bio.h>
11#include <linux/blkdev.h>
12#include <linux/prefetch.h>
13#include <linux/kthread.h>
14#include <linux/swap.h>
15#include <linux/timer.h>
16#include <linux/freezer.h>
17#include <linux/sched/signal.h>
18
19#include "f2fs.h"
20#include "segment.h"
21#include "node.h"
22#include "gc.h"
23#include <trace/events/f2fs.h>
24
25#define __reverse_ffz(x) __reverse_ffs(~(x))
26
27static struct kmem_cache *discard_entry_slab;
28static struct kmem_cache *discard_cmd_slab;
29static struct kmem_cache *sit_entry_set_slab;
30static struct kmem_cache *inmem_entry_slab;
31
32static unsigned long __reverse_ulong(unsigned char *str)
33{
34 unsigned long tmp = 0;
35 int shift = 24, idx = 0;
36
37#if BITS_PER_LONG == 64
38 shift = 56;
39#endif
40 while (shift >= 0) {
41 tmp |= (unsigned long)str[idx++] << shift;
42 shift -= BITS_PER_BYTE;
43 }
44 return tmp;
45}
46
47
48
49
50
51static inline unsigned long __reverse_ffs(unsigned long word)
52{
53 int num = 0;
54
55#if BITS_PER_LONG == 64
56 if ((word & 0xffffffff00000000UL) == 0)
57 num += 32;
58 else
59 word >>= 32;
60#endif
61 if ((word & 0xffff0000) == 0)
62 num += 16;
63 else
64 word >>= 16;
65
66 if ((word & 0xff00) == 0)
67 num += 8;
68 else
69 word >>= 8;
70
71 if ((word & 0xf0) == 0)
72 num += 4;
73 else
74 word >>= 4;
75
76 if ((word & 0xc) == 0)
77 num += 2;
78 else
79 word >>= 2;
80
81 if ((word & 0x2) == 0)
82 num += 1;
83 return num;
84}
85
86
87
88
89
90
91
92
93
94
95static unsigned long __find_rev_next_bit(const unsigned long *addr,
96 unsigned long size, unsigned long offset)
97{
98 const unsigned long *p = addr + BIT_WORD(offset);
99 unsigned long result = size;
100 unsigned long tmp;
101
102 if (offset >= size)
103 return size;
104
105 size -= (offset & ~(BITS_PER_LONG - 1));
106 offset %= BITS_PER_LONG;
107
108 while (1) {
109 if (*p == 0)
110 goto pass;
111
112 tmp = __reverse_ulong((unsigned char *)p);
113
114 tmp &= ~0UL >> offset;
115 if (size < BITS_PER_LONG)
116 tmp &= (~0UL << (BITS_PER_LONG - size));
117 if (tmp)
118 goto found;
119pass:
120 if (size <= BITS_PER_LONG)
121 break;
122 size -= BITS_PER_LONG;
123 offset = 0;
124 p++;
125 }
126 return result;
127found:
128 return result - size + __reverse_ffs(tmp);
129}
130
131static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
132 unsigned long size, unsigned long offset)
133{
134 const unsigned long *p = addr + BIT_WORD(offset);
135 unsigned long result = size;
136 unsigned long tmp;
137
138 if (offset >= size)
139 return size;
140
141 size -= (offset & ~(BITS_PER_LONG - 1));
142 offset %= BITS_PER_LONG;
143
144 while (1) {
145 if (*p == ~0UL)
146 goto pass;
147
148 tmp = __reverse_ulong((unsigned char *)p);
149
150 if (offset)
151 tmp |= ~0UL << (BITS_PER_LONG - offset);
152 if (size < BITS_PER_LONG)
153 tmp |= ~0UL >> size;
154 if (tmp != ~0UL)
155 goto found;
156pass:
157 if (size <= BITS_PER_LONG)
158 break;
159 size -= BITS_PER_LONG;
160 offset = 0;
161 p++;
162 }
163 return result;
164found:
165 return result - size + __reverse_ffz(tmp);
166}
167
168bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
169{
170 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
171 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
172 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
173
174 if (f2fs_lfs_mode(sbi))
175 return false;
176 if (sbi->gc_mode == GC_URGENT_HIGH)
177 return true;
178 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
179 return true;
180
181 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
182 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
183}
184
185void f2fs_register_inmem_page(struct inode *inode, struct page *page)
186{
187 struct inmem_pages *new;
188
189 if (PagePrivate(page))
190 set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE);
191 else
192 f2fs_set_page_private(page, ATOMIC_WRITTEN_PAGE);
193
194 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
195
196
197 new->page = page;
198 INIT_LIST_HEAD(&new->list);
199
200
201 get_page(page);
202 mutex_lock(&F2FS_I(inode)->inmem_lock);
203 list_add_tail(&new->list, &F2FS_I(inode)->inmem_pages);
204 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
205 mutex_unlock(&F2FS_I(inode)->inmem_lock);
206
207 trace_f2fs_register_inmem_page(page, INMEM);
208}
209
210static int __revoke_inmem_pages(struct inode *inode,
211 struct list_head *head, bool drop, bool recover,
212 bool trylock)
213{
214 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
215 struct inmem_pages *cur, *tmp;
216 int err = 0;
217
218 list_for_each_entry_safe(cur, tmp, head, list) {
219 struct page *page = cur->page;
220
221 if (drop)
222 trace_f2fs_commit_inmem_page(page, INMEM_DROP);
223
224 if (trylock) {
225
226
227
228
229 if (!trylock_page(page))
230 continue;
231 } else {
232 lock_page(page);
233 }
234
235 f2fs_wait_on_page_writeback(page, DATA, true, true);
236
237 if (recover) {
238 struct dnode_of_data dn;
239 struct node_info ni;
240
241 trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
242retry:
243 set_new_dnode(&dn, inode, NULL, NULL, 0);
244 err = f2fs_get_dnode_of_data(&dn, page->index,
245 LOOKUP_NODE);
246 if (err) {
247 if (err == -ENOMEM) {
248 congestion_wait(BLK_RW_ASYNC,
249 DEFAULT_IO_TIMEOUT);
250 cond_resched();
251 goto retry;
252 }
253 err = -EAGAIN;
254 goto next;
255 }
256
257 err = f2fs_get_node_info(sbi, dn.nid, &ni);
258 if (err) {
259 f2fs_put_dnode(&dn);
260 return err;
261 }
262
263 if (cur->old_addr == NEW_ADDR) {
264 f2fs_invalidate_blocks(sbi, dn.data_blkaddr);
265 f2fs_update_data_blkaddr(&dn, NEW_ADDR);
266 } else
267 f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
268 cur->old_addr, ni.version, true, true);
269 f2fs_put_dnode(&dn);
270 }
271next:
272
273 if (drop || recover) {
274 ClearPageUptodate(page);
275 clear_cold_data(page);
276 }
277 f2fs_clear_page_private(page);
278 f2fs_put_page(page, 1);
279
280 list_del(&cur->list);
281 kmem_cache_free(inmem_entry_slab, cur);
282 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
283 }
284 return err;
285}
286
287void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)
288{
289 struct list_head *head = &sbi->inode_list[ATOMIC_FILE];
290 struct inode *inode;
291 struct f2fs_inode_info *fi;
292 unsigned int count = sbi->atomic_files;
293 unsigned int looped = 0;
294next:
295 spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
296 if (list_empty(head)) {
297 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
298 return;
299 }
300 fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist);
301 inode = igrab(&fi->vfs_inode);
302 if (inode)
303 list_move_tail(&fi->inmem_ilist, head);
304 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
305
306 if (inode) {
307 if (gc_failure) {
308 if (!fi->i_gc_failures[GC_FAILURE_ATOMIC])
309 goto skip;
310 }
311 set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST);
312 f2fs_drop_inmem_pages(inode);
313skip:
314 iput(inode);
315 }
316 congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
317 cond_resched();
318 if (gc_failure) {
319 if (++looped >= count)
320 return;
321 }
322 goto next;
323}
324
325void f2fs_drop_inmem_pages(struct inode *inode)
326{
327 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
328 struct f2fs_inode_info *fi = F2FS_I(inode);
329
330 do {
331 mutex_lock(&fi->inmem_lock);
332 if (list_empty(&fi->inmem_pages)) {
333 fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0;
334
335 spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
336 if (!list_empty(&fi->inmem_ilist))
337 list_del_init(&fi->inmem_ilist);
338 if (f2fs_is_atomic_file(inode)) {
339 clear_inode_flag(inode, FI_ATOMIC_FILE);
340 sbi->atomic_files--;
341 }
342 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
343
344 mutex_unlock(&fi->inmem_lock);
345 break;
346 }
347 __revoke_inmem_pages(inode, &fi->inmem_pages,
348 true, false, true);
349 mutex_unlock(&fi->inmem_lock);
350 } while (1);
351}
352
353void f2fs_drop_inmem_page(struct inode *inode, struct page *page)
354{
355 struct f2fs_inode_info *fi = F2FS_I(inode);
356 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
357 struct list_head *head = &fi->inmem_pages;
358 struct inmem_pages *cur = NULL;
359
360 f2fs_bug_on(sbi, !IS_ATOMIC_WRITTEN_PAGE(page));
361
362 mutex_lock(&fi->inmem_lock);
363 list_for_each_entry(cur, head, list) {
364 if (cur->page == page)
365 break;
366 }
367
368 f2fs_bug_on(sbi, list_empty(head) || cur->page != page);
369 list_del(&cur->list);
370 mutex_unlock(&fi->inmem_lock);
371
372 dec_page_count(sbi, F2FS_INMEM_PAGES);
373 kmem_cache_free(inmem_entry_slab, cur);
374
375 ClearPageUptodate(page);
376 f2fs_clear_page_private(page);
377 f2fs_put_page(page, 0);
378
379 trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE);
380}
381
382static int __f2fs_commit_inmem_pages(struct inode *inode)
383{
384 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
385 struct f2fs_inode_info *fi = F2FS_I(inode);
386 struct inmem_pages *cur, *tmp;
387 struct f2fs_io_info fio = {
388 .sbi = sbi,
389 .ino = inode->i_ino,
390 .type = DATA,
391 .op = REQ_OP_WRITE,
392 .op_flags = REQ_SYNC | REQ_PRIO,
393 .io_type = FS_DATA_IO,
394 };
395 struct list_head revoke_list;
396 bool submit_bio = false;
397 int err = 0;
398
399 INIT_LIST_HEAD(&revoke_list);
400
401 list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
402 struct page *page = cur->page;
403
404 lock_page(page);
405 if (page->mapping == inode->i_mapping) {
406 trace_f2fs_commit_inmem_page(page, INMEM);
407
408 f2fs_wait_on_page_writeback(page, DATA, true, true);
409
410 set_page_dirty(page);
411 if (clear_page_dirty_for_io(page)) {
412 inode_dec_dirty_pages(inode);
413 f2fs_remove_dirty_inode(inode);
414 }
415retry:
416 fio.page = page;
417 fio.old_blkaddr = NULL_ADDR;
418 fio.encrypted_page = NULL;
419 fio.need_lock = LOCK_DONE;
420 err = f2fs_do_write_data_page(&fio);
421 if (err) {
422 if (err == -ENOMEM) {
423 congestion_wait(BLK_RW_ASYNC,
424 DEFAULT_IO_TIMEOUT);
425 cond_resched();
426 goto retry;
427 }
428 unlock_page(page);
429 break;
430 }
431
432 cur->old_addr = fio.old_blkaddr;
433 submit_bio = true;
434 }
435 unlock_page(page);
436 list_move_tail(&cur->list, &revoke_list);
437 }
438
439 if (submit_bio)
440 f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA);
441
442 if (err) {
443
444
445
446
447
448
449
450
451 err = __revoke_inmem_pages(inode, &revoke_list,
452 false, true, false);
453
454
455 __revoke_inmem_pages(inode, &fi->inmem_pages,
456 true, false, false);
457 } else {
458 __revoke_inmem_pages(inode, &revoke_list,
459 false, false, false);
460 }
461
462 return err;
463}
464
465int f2fs_commit_inmem_pages(struct inode *inode)
466{
467 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
468 struct f2fs_inode_info *fi = F2FS_I(inode);
469 int err;
470
471 f2fs_balance_fs(sbi, true);
472
473 down_write(&fi->i_gc_rwsem[WRITE]);
474
475 f2fs_lock_op(sbi);
476 set_inode_flag(inode, FI_ATOMIC_COMMIT);
477
478 mutex_lock(&fi->inmem_lock);
479 err = __f2fs_commit_inmem_pages(inode);
480 mutex_unlock(&fi->inmem_lock);
481
482 clear_inode_flag(inode, FI_ATOMIC_COMMIT);
483
484 f2fs_unlock_op(sbi);
485 up_write(&fi->i_gc_rwsem[WRITE]);
486
487 return err;
488}
489
490
491
492
493
494void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
495{
496 if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
497 f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
498 f2fs_stop_checkpoint(sbi, false);
499 }
500
501
502 if (need && excess_cached_nats(sbi))
503 f2fs_balance_fs_bg(sbi, false);
504
505 if (!f2fs_is_checkpoint_ready(sbi))
506 return;
507
508
509
510
511
512 if (has_not_enough_free_secs(sbi, 0, 0)) {
513 if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
514 sbi->gc_thread->f2fs_gc_task) {
515 DEFINE_WAIT(wait);
516
517 prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
518 TASK_UNINTERRUPTIBLE);
519 wake_up(&sbi->gc_thread->gc_wait_queue_head);
520 io_schedule();
521 finish_wait(&sbi->gc_thread->fggc_wq, &wait);
522 } else {
523 down_write(&sbi->gc_lock);
524 f2fs_gc(sbi, false, false, false, NULL_SEGNO);
525 }
526 }
527}
528
529void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
530{
531 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
532 return;
533
534
535 if (!f2fs_available_free_memory(sbi, EXTENT_CACHE))
536 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
537
538
539 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
540 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
541
542 if (!f2fs_available_free_memory(sbi, FREE_NIDS))
543 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
544 else
545 f2fs_build_free_nids(sbi, false, false);
546
547 if (excess_dirty_nats(sbi) || excess_dirty_nodes(sbi) ||
548 excess_prefree_segs(sbi))
549 goto do_sync;
550
551
552 if (is_inflight_io(sbi, REQ_TIME) ||
553 (!f2fs_time_over(sbi, REQ_TIME) && rwsem_is_locked(&sbi->cp_rwsem)))
554 return;
555
556
557 if (f2fs_time_over(sbi, CP_TIME))
558 goto do_sync;
559
560
561 if (f2fs_available_free_memory(sbi, NAT_ENTRIES) ||
562 f2fs_available_free_memory(sbi, INO_ENTRIES))
563 return;
564
565do_sync:
566 if (test_opt(sbi, DATA_FLUSH) && from_bg) {
567 struct blk_plug plug;
568
569 mutex_lock(&sbi->flush_lock);
570
571 blk_start_plug(&plug);
572 f2fs_sync_dirty_inodes(sbi, FILE_INODE);
573 blk_finish_plug(&plug);
574
575 mutex_unlock(&sbi->flush_lock);
576 }
577 f2fs_sync_fs(sbi->sb, true);
578 stat_inc_bg_cp_count(sbi->stat_info);
579}
580
581static int __submit_flush_wait(struct f2fs_sb_info *sbi,
582 struct block_device *bdev)
583{
584 int ret = blkdev_issue_flush(bdev);
585
586 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
587 test_opt(sbi, FLUSH_MERGE), ret);
588 return ret;
589}
590
591static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
592{
593 int ret = 0;
594 int i;
595
596 if (!f2fs_is_multi_device(sbi))
597 return __submit_flush_wait(sbi, sbi->sb->s_bdev);
598
599 for (i = 0; i < sbi->s_ndevs; i++) {
600 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
601 continue;
602 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
603 if (ret)
604 break;
605 }
606 return ret;
607}
608
609static int issue_flush_thread(void *data)
610{
611 struct f2fs_sb_info *sbi = data;
612 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
613 wait_queue_head_t *q = &fcc->flush_wait_queue;
614repeat:
615 if (kthread_should_stop())
616 return 0;
617
618 if (!llist_empty(&fcc->issue_list)) {
619 struct flush_cmd *cmd, *next;
620 int ret;
621
622 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
623 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
624
625 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
626
627 ret = submit_flush_wait(sbi, cmd->ino);
628 atomic_inc(&fcc->issued_flush);
629
630 llist_for_each_entry_safe(cmd, next,
631 fcc->dispatch_list, llnode) {
632 cmd->ret = ret;
633 complete(&cmd->wait);
634 }
635 fcc->dispatch_list = NULL;
636 }
637
638 wait_event_interruptible(*q,
639 kthread_should_stop() || !llist_empty(&fcc->issue_list));
640 goto repeat;
641}
642
643int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
644{
645 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
646 struct flush_cmd cmd;
647 int ret;
648
649 if (test_opt(sbi, NOBARRIER))
650 return 0;
651
652 if (!test_opt(sbi, FLUSH_MERGE)) {
653 atomic_inc(&fcc->queued_flush);
654 ret = submit_flush_wait(sbi, ino);
655 atomic_dec(&fcc->queued_flush);
656 atomic_inc(&fcc->issued_flush);
657 return ret;
658 }
659
660 if (atomic_inc_return(&fcc->queued_flush) == 1 ||
661 f2fs_is_multi_device(sbi)) {
662 ret = submit_flush_wait(sbi, ino);
663 atomic_dec(&fcc->queued_flush);
664
665 atomic_inc(&fcc->issued_flush);
666 return ret;
667 }
668
669 cmd.ino = ino;
670 init_completion(&cmd.wait);
671
672 llist_add(&cmd.llnode, &fcc->issue_list);
673
674
675
676
677
678
679 smp_mb();
680
681 if (waitqueue_active(&fcc->flush_wait_queue))
682 wake_up(&fcc->flush_wait_queue);
683
684 if (fcc->f2fs_issue_flush) {
685 wait_for_completion(&cmd.wait);
686 atomic_dec(&fcc->queued_flush);
687 } else {
688 struct llist_node *list;
689
690 list = llist_del_all(&fcc->issue_list);
691 if (!list) {
692 wait_for_completion(&cmd.wait);
693 atomic_dec(&fcc->queued_flush);
694 } else {
695 struct flush_cmd *tmp, *next;
696
697 ret = submit_flush_wait(sbi, ino);
698
699 llist_for_each_entry_safe(tmp, next, list, llnode) {
700 if (tmp == &cmd) {
701 cmd.ret = ret;
702 atomic_dec(&fcc->queued_flush);
703 continue;
704 }
705 tmp->ret = ret;
706 complete(&tmp->wait);
707 }
708 }
709 }
710
711 return cmd.ret;
712}
713
714int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
715{
716 dev_t dev = sbi->sb->s_bdev->bd_dev;
717 struct flush_cmd_control *fcc;
718 int err = 0;
719
720 if (SM_I(sbi)->fcc_info) {
721 fcc = SM_I(sbi)->fcc_info;
722 if (fcc->f2fs_issue_flush)
723 return err;
724 goto init_thread;
725 }
726
727 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
728 if (!fcc)
729 return -ENOMEM;
730 atomic_set(&fcc->issued_flush, 0);
731 atomic_set(&fcc->queued_flush, 0);
732 init_waitqueue_head(&fcc->flush_wait_queue);
733 init_llist_head(&fcc->issue_list);
734 SM_I(sbi)->fcc_info = fcc;
735 if (!test_opt(sbi, FLUSH_MERGE))
736 return err;
737
738init_thread:
739 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
740 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
741 if (IS_ERR(fcc->f2fs_issue_flush)) {
742 err = PTR_ERR(fcc->f2fs_issue_flush);
743 kfree(fcc);
744 SM_I(sbi)->fcc_info = NULL;
745 return err;
746 }
747
748 return err;
749}
750
751void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
752{
753 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
754
755 if (fcc && fcc->f2fs_issue_flush) {
756 struct task_struct *flush_thread = fcc->f2fs_issue_flush;
757
758 fcc->f2fs_issue_flush = NULL;
759 kthread_stop(flush_thread);
760 }
761 if (free) {
762 kfree(fcc);
763 SM_I(sbi)->fcc_info = NULL;
764 }
765}
766
767int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
768{
769 int ret = 0, i;
770
771 if (!f2fs_is_multi_device(sbi))
772 return 0;
773
774 if (test_opt(sbi, NOBARRIER))
775 return 0;
776
777 for (i = 1; i < sbi->s_ndevs; i++) {
778 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
779 continue;
780 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
781 if (ret)
782 break;
783
784 spin_lock(&sbi->dev_lock);
785 f2fs_clear_bit(i, (char *)&sbi->dirty_device);
786 spin_unlock(&sbi->dev_lock);
787 }
788
789 return ret;
790}
791
792static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
793 enum dirty_type dirty_type)
794{
795 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
796
797
798 if (IS_CURSEG(sbi, segno))
799 return;
800
801 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
802 dirty_i->nr_dirty[dirty_type]++;
803
804 if (dirty_type == DIRTY) {
805 struct seg_entry *sentry = get_seg_entry(sbi, segno);
806 enum dirty_type t = sentry->type;
807
808 if (unlikely(t >= DIRTY)) {
809 f2fs_bug_on(sbi, 1);
810 return;
811 }
812 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
813 dirty_i->nr_dirty[t]++;
814
815 if (__is_large_section(sbi)) {
816 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
817 block_t valid_blocks =
818 get_valid_blocks(sbi, segno, true);
819
820 f2fs_bug_on(sbi, unlikely(!valid_blocks ||
821 valid_blocks == BLKS_PER_SEC(sbi)));
822
823 if (!IS_CURSEC(sbi, secno))
824 set_bit(secno, dirty_i->dirty_secmap);
825 }
826 }
827}
828
829static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
830 enum dirty_type dirty_type)
831{
832 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
833 block_t valid_blocks;
834
835 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
836 dirty_i->nr_dirty[dirty_type]--;
837
838 if (dirty_type == DIRTY) {
839 struct seg_entry *sentry = get_seg_entry(sbi, segno);
840 enum dirty_type t = sentry->type;
841
842 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
843 dirty_i->nr_dirty[t]--;
844
845 valid_blocks = get_valid_blocks(sbi, segno, true);
846 if (valid_blocks == 0) {
847 clear_bit(GET_SEC_FROM_SEG(sbi, segno),
848 dirty_i->victim_secmap);
849#ifdef CONFIG_F2FS_CHECK_FS
850 clear_bit(segno, SIT_I(sbi)->invalid_segmap);
851#endif
852 }
853 if (__is_large_section(sbi)) {
854 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
855
856 if (!valid_blocks ||
857 valid_blocks == BLKS_PER_SEC(sbi)) {
858 clear_bit(secno, dirty_i->dirty_secmap);
859 return;
860 }
861
862 if (!IS_CURSEC(sbi, secno))
863 set_bit(secno, dirty_i->dirty_secmap);
864 }
865 }
866}
867
868
869
870
871
872
873static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
874{
875 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
876 unsigned short valid_blocks, ckpt_valid_blocks;
877 unsigned int usable_blocks;
878
879 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
880 return;
881
882 usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
883 mutex_lock(&dirty_i->seglist_lock);
884
885 valid_blocks = get_valid_blocks(sbi, segno, false);
886 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
887
888 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
889 ckpt_valid_blocks == usable_blocks)) {
890 __locate_dirty_segment(sbi, segno, PRE);
891 __remove_dirty_segment(sbi, segno, DIRTY);
892 } else if (valid_blocks < usable_blocks) {
893 __locate_dirty_segment(sbi, segno, DIRTY);
894 } else {
895
896 __remove_dirty_segment(sbi, segno, DIRTY);
897 }
898
899 mutex_unlock(&dirty_i->seglist_lock);
900}
901
902
903void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
904{
905 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
906 unsigned int segno;
907
908 mutex_lock(&dirty_i->seglist_lock);
909 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
910 if (get_valid_blocks(sbi, segno, false))
911 continue;
912 if (IS_CURSEG(sbi, segno))
913 continue;
914 __locate_dirty_segment(sbi, segno, PRE);
915 __remove_dirty_segment(sbi, segno, DIRTY);
916 }
917 mutex_unlock(&dirty_i->seglist_lock);
918}
919
920block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
921{
922 int ovp_hole_segs =
923 (overprovision_segments(sbi) - reserved_segments(sbi));
924 block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;
925 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
926 block_t holes[2] = {0, 0};
927 block_t unusable;
928 struct seg_entry *se;
929 unsigned int segno;
930
931 mutex_lock(&dirty_i->seglist_lock);
932 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
933 se = get_seg_entry(sbi, segno);
934 if (IS_NODESEG(se->type))
935 holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
936 se->valid_blocks;
937 else
938 holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
939 se->valid_blocks;
940 }
941 mutex_unlock(&dirty_i->seglist_lock);
942
943 unusable = holes[DATA] > holes[NODE] ? holes[DATA] : holes[NODE];
944 if (unusable > ovp_holes)
945 return unusable - ovp_holes;
946 return 0;
947}
948
949int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
950{
951 int ovp_hole_segs =
952 (overprovision_segments(sbi) - reserved_segments(sbi));
953 if (unusable > F2FS_OPTION(sbi).unusable_cap)
954 return -EAGAIN;
955 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
956 dirty_segments(sbi) > ovp_hole_segs)
957 return -EAGAIN;
958 return 0;
959}
960
961
962static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
963{
964 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
965 unsigned int segno = 0;
966
967 mutex_lock(&dirty_i->seglist_lock);
968 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
969 if (get_valid_blocks(sbi, segno, false))
970 continue;
971 if (get_ckpt_valid_blocks(sbi, segno, false))
972 continue;
973 mutex_unlock(&dirty_i->seglist_lock);
974 return segno;
975 }
976 mutex_unlock(&dirty_i->seglist_lock);
977 return NULL_SEGNO;
978}
979
980static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
981 struct block_device *bdev, block_t lstart,
982 block_t start, block_t len)
983{
984 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
985 struct list_head *pend_list;
986 struct discard_cmd *dc;
987
988 f2fs_bug_on(sbi, !len);
989
990 pend_list = &dcc->pend_list[plist_idx(len)];
991
992 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS);
993 INIT_LIST_HEAD(&dc->list);
994 dc->bdev = bdev;
995 dc->lstart = lstart;
996 dc->start = start;
997 dc->len = len;
998 dc->ref = 0;
999 dc->state = D_PREP;
1000 dc->queued = 0;
1001 dc->error = 0;
1002 init_completion(&dc->wait);
1003 list_add_tail(&dc->list, pend_list);
1004 spin_lock_init(&dc->lock);
1005 dc->bio_ref = 0;
1006 atomic_inc(&dcc->discard_cmd_cnt);
1007 dcc->undiscard_blks += len;
1008
1009 return dc;
1010}
1011
1012static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,
1013 struct block_device *bdev, block_t lstart,
1014 block_t start, block_t len,
1015 struct rb_node *parent, struct rb_node **p,
1016 bool leftmost)
1017{
1018 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1019 struct discard_cmd *dc;
1020
1021 dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
1022
1023 rb_link_node(&dc->rb_node, parent, p);
1024 rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
1025
1026 return dc;
1027}
1028
1029static void __detach_discard_cmd(struct discard_cmd_control *dcc,
1030 struct discard_cmd *dc)
1031{
1032 if (dc->state == D_DONE)
1033 atomic_sub(dc->queued, &dcc->queued_discard);
1034
1035 list_del(&dc->list);
1036 rb_erase_cached(&dc->rb_node, &dcc->root);
1037 dcc->undiscard_blks -= dc->len;
1038
1039 kmem_cache_free(discard_cmd_slab, dc);
1040
1041 atomic_dec(&dcc->discard_cmd_cnt);
1042}
1043
1044static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
1045 struct discard_cmd *dc)
1046{
1047 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1048 unsigned long flags;
1049
1050 trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len);
1051
1052 spin_lock_irqsave(&dc->lock, flags);
1053 if (dc->bio_ref) {
1054 spin_unlock_irqrestore(&dc->lock, flags);
1055 return;
1056 }
1057 spin_unlock_irqrestore(&dc->lock, flags);
1058
1059 f2fs_bug_on(sbi, dc->ref);
1060
1061 if (dc->error == -EOPNOTSUPP)
1062 dc->error = 0;
1063
1064 if (dc->error)
1065 printk_ratelimited(
1066 "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d",
1067 KERN_INFO, sbi->sb->s_id,
1068 dc->lstart, dc->start, dc->len, dc->error);
1069 __detach_discard_cmd(dcc, dc);
1070}
1071
1072static void f2fs_submit_discard_endio(struct bio *bio)
1073{
1074 struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
1075 unsigned long flags;
1076
1077 spin_lock_irqsave(&dc->lock, flags);
1078 if (!dc->error)
1079 dc->error = blk_status_to_errno(bio->bi_status);
1080 dc->bio_ref--;
1081 if (!dc->bio_ref && dc->state == D_SUBMIT) {
1082 dc->state = D_DONE;
1083 complete_all(&dc->wait);
1084 }
1085 spin_unlock_irqrestore(&dc->lock, flags);
1086 bio_put(bio);
1087}
1088
1089static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
1090 block_t start, block_t end)
1091{
1092#ifdef CONFIG_F2FS_CHECK_FS
1093 struct seg_entry *sentry;
1094 unsigned int segno;
1095 block_t blk = start;
1096 unsigned long offset, size, max_blocks = sbi->blocks_per_seg;
1097 unsigned long *map;
1098
1099 while (blk < end) {
1100 segno = GET_SEGNO(sbi, blk);
1101 sentry = get_seg_entry(sbi, segno);
1102 offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
1103
1104 if (end < START_BLOCK(sbi, segno + 1))
1105 size = GET_BLKOFF_FROM_SEG0(sbi, end);
1106 else
1107 size = max_blocks;
1108 map = (unsigned long *)(sentry->cur_valid_map);
1109 offset = __find_rev_next_bit(map, size, offset);
1110 f2fs_bug_on(sbi, offset != size);
1111 blk = START_BLOCK(sbi, segno + 1);
1112 }
1113#endif
1114}
1115
1116static void __init_discard_policy(struct f2fs_sb_info *sbi,
1117 struct discard_policy *dpolicy,
1118 int discard_type, unsigned int granularity)
1119{
1120 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1121
1122
1123 dpolicy->type = discard_type;
1124 dpolicy->sync = true;
1125 dpolicy->ordered = false;
1126 dpolicy->granularity = granularity;
1127
1128 dpolicy->max_requests = DEF_MAX_DISCARD_REQUEST;
1129 dpolicy->io_aware_gran = MAX_PLIST_NUM;
1130 dpolicy->timeout = false;
1131
1132 if (discard_type == DPOLICY_BG) {
1133 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
1134 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
1135 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
1136 dpolicy->io_aware = true;
1137 dpolicy->sync = false;
1138 dpolicy->ordered = true;
1139 if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {
1140 dpolicy->granularity = 1;
1141 if (atomic_read(&dcc->discard_cmd_cnt))
1142 dpolicy->max_interval =
1143 DEF_MIN_DISCARD_ISSUE_TIME;
1144 }
1145 } else if (discard_type == DPOLICY_FORCE) {
1146 dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
1147 dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
1148 dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
1149 dpolicy->io_aware = false;
1150 } else if (discard_type == DPOLICY_FSTRIM) {
1151 dpolicy->io_aware = false;
1152 } else if (discard_type == DPOLICY_UMOUNT) {
1153 dpolicy->io_aware = false;
1154
1155 dpolicy->granularity = 1;
1156 dpolicy->timeout = true;
1157 }
1158}
1159
1160static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1161 struct block_device *bdev, block_t lstart,
1162 block_t start, block_t len);
1163
1164static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
1165 struct discard_policy *dpolicy,
1166 struct discard_cmd *dc,
1167 unsigned int *issued)
1168{
1169 struct block_device *bdev = dc->bdev;
1170 struct request_queue *q = bdev_get_queue(bdev);
1171 unsigned int max_discard_blocks =
1172 SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
1173 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1174 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1175 &(dcc->fstrim_list) : &(dcc->wait_list);
1176 int flag = dpolicy->sync ? REQ_SYNC : 0;
1177 block_t lstart, start, len, total_len;
1178 int err = 0;
1179
1180 if (dc->state != D_PREP)
1181 return 0;
1182
1183 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1184 return 0;
1185
1186 trace_f2fs_issue_discard(bdev, dc->start, dc->len);
1187
1188 lstart = dc->lstart;
1189 start = dc->start;
1190 len = dc->len;
1191 total_len = len;
1192
1193 dc->len = 0;
1194
1195 while (total_len && *issued < dpolicy->max_requests && !err) {
1196 struct bio *bio = NULL;
1197 unsigned long flags;
1198 bool last = true;
1199
1200 if (len > max_discard_blocks) {
1201 len = max_discard_blocks;
1202 last = false;
1203 }
1204
1205 (*issued)++;
1206 if (*issued == dpolicy->max_requests)
1207 last = true;
1208
1209 dc->len += len;
1210
1211 if (time_to_inject(sbi, FAULT_DISCARD)) {
1212 f2fs_show_injection_info(sbi, FAULT_DISCARD);
1213 err = -EIO;
1214 goto submit;
1215 }
1216 err = __blkdev_issue_discard(bdev,
1217 SECTOR_FROM_BLOCK(start),
1218 SECTOR_FROM_BLOCK(len),
1219 GFP_NOFS, 0, &bio);
1220submit:
1221 if (err) {
1222 spin_lock_irqsave(&dc->lock, flags);
1223 if (dc->state == D_PARTIAL)
1224 dc->state = D_SUBMIT;
1225 spin_unlock_irqrestore(&dc->lock, flags);
1226
1227 break;
1228 }
1229
1230 f2fs_bug_on(sbi, !bio);
1231
1232
1233
1234
1235
1236 spin_lock_irqsave(&dc->lock, flags);
1237 if (last)
1238 dc->state = D_SUBMIT;
1239 else
1240 dc->state = D_PARTIAL;
1241 dc->bio_ref++;
1242 spin_unlock_irqrestore(&dc->lock, flags);
1243
1244 atomic_inc(&dcc->queued_discard);
1245 dc->queued++;
1246 list_move_tail(&dc->list, wait_list);
1247
1248
1249 __check_sit_bitmap(sbi, lstart, lstart + len);
1250
1251 bio->bi_private = dc;
1252 bio->bi_end_io = f2fs_submit_discard_endio;
1253 bio->bi_opf |= flag;
1254 submit_bio(bio);
1255
1256 atomic_inc(&dcc->issued_discard);
1257
1258 f2fs_update_iostat(sbi, FS_DISCARD, 1);
1259
1260 lstart += len;
1261 start += len;
1262 total_len -= len;
1263 len = total_len;
1264 }
1265
1266 if (!err && len) {
1267 dcc->undiscard_blks -= len;
1268 __update_discard_tree_range(sbi, bdev, lstart, start, len);
1269 }
1270 return err;
1271}
1272
1273static void __insert_discard_tree(struct f2fs_sb_info *sbi,
1274 struct block_device *bdev, block_t lstart,
1275 block_t start, block_t len,
1276 struct rb_node **insert_p,
1277 struct rb_node *insert_parent)
1278{
1279 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1280 struct rb_node **p;
1281 struct rb_node *parent = NULL;
1282 bool leftmost = true;
1283
1284 if (insert_p && insert_parent) {
1285 parent = insert_parent;
1286 p = insert_p;
1287 goto do_insert;
1288 }
1289
1290 p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,
1291 lstart, &leftmost);
1292do_insert:
1293 __attach_discard_cmd(sbi, bdev, lstart, start, len, parent,
1294 p, leftmost);
1295}
1296
1297static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
1298 struct discard_cmd *dc)
1299{
1300 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);
1301}
1302
1303static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
1304 struct discard_cmd *dc, block_t blkaddr)
1305{
1306 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1307 struct discard_info di = dc->di;
1308 bool modified = false;
1309
1310 if (dc->state == D_DONE || dc->len == 1) {
1311 __remove_discard_cmd(sbi, dc);
1312 return;
1313 }
1314
1315 dcc->undiscard_blks -= di.len;
1316
1317 if (blkaddr > di.lstart) {
1318 dc->len = blkaddr - dc->lstart;
1319 dcc->undiscard_blks += dc->len;
1320 __relocate_discard_cmd(dcc, dc);
1321 modified = true;
1322 }
1323
1324 if (blkaddr < di.lstart + di.len - 1) {
1325 if (modified) {
1326 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
1327 di.start + blkaddr + 1 - di.lstart,
1328 di.lstart + di.len - 1 - blkaddr,
1329 NULL, NULL);
1330 } else {
1331 dc->lstart++;
1332 dc->len--;
1333 dc->start++;
1334 dcc->undiscard_blks += dc->len;
1335 __relocate_discard_cmd(dcc, dc);
1336 }
1337 }
1338}
1339
1340static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1341 struct block_device *bdev, block_t lstart,
1342 block_t start, block_t len)
1343{
1344 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1345 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1346 struct discard_cmd *dc;
1347 struct discard_info di = {0};
1348 struct rb_node **insert_p = NULL, *insert_parent = NULL;
1349 struct request_queue *q = bdev_get_queue(bdev);
1350 unsigned int max_discard_blocks =
1351 SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
1352 block_t end = lstart + len;
1353
1354 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1355 NULL, lstart,
1356 (struct rb_entry **)&prev_dc,
1357 (struct rb_entry **)&next_dc,
1358 &insert_p, &insert_parent, true, NULL);
1359 if (dc)
1360 prev_dc = dc;
1361
1362 if (!prev_dc) {
1363 di.lstart = lstart;
1364 di.len = next_dc ? next_dc->lstart - lstart : len;
1365 di.len = min(di.len, len);
1366 di.start = start;
1367 }
1368
1369 while (1) {
1370 struct rb_node *node;
1371 bool merged = false;
1372 struct discard_cmd *tdc = NULL;
1373
1374 if (prev_dc) {
1375 di.lstart = prev_dc->lstart + prev_dc->len;
1376 if (di.lstart < lstart)
1377 di.lstart = lstart;
1378 if (di.lstart >= end)
1379 break;
1380
1381 if (!next_dc || next_dc->lstart > end)
1382 di.len = end - di.lstart;
1383 else
1384 di.len = next_dc->lstart - di.lstart;
1385 di.start = start + di.lstart - lstart;
1386 }
1387
1388 if (!di.len)
1389 goto next;
1390
1391 if (prev_dc && prev_dc->state == D_PREP &&
1392 prev_dc->bdev == bdev &&
1393 __is_discard_back_mergeable(&di, &prev_dc->di,
1394 max_discard_blocks)) {
1395 prev_dc->di.len += di.len;
1396 dcc->undiscard_blks += di.len;
1397 __relocate_discard_cmd(dcc, prev_dc);
1398 di = prev_dc->di;
1399 tdc = prev_dc;
1400 merged = true;
1401 }
1402
1403 if (next_dc && next_dc->state == D_PREP &&
1404 next_dc->bdev == bdev &&
1405 __is_discard_front_mergeable(&di, &next_dc->di,
1406 max_discard_blocks)) {
1407 next_dc->di.lstart = di.lstart;
1408 next_dc->di.len += di.len;
1409 next_dc->di.start = di.start;
1410 dcc->undiscard_blks += di.len;
1411 __relocate_discard_cmd(dcc, next_dc);
1412 if (tdc)
1413 __remove_discard_cmd(sbi, tdc);
1414 merged = true;
1415 }
1416
1417 if (!merged) {
1418 __insert_discard_tree(sbi, bdev, di.lstart, di.start,
1419 di.len, NULL, NULL);
1420 }
1421 next:
1422 prev_dc = next_dc;
1423 if (!prev_dc)
1424 break;
1425
1426 node = rb_next(&prev_dc->rb_node);
1427 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1428 }
1429}
1430
1431static int __queue_discard_cmd(struct f2fs_sb_info *sbi,
1432 struct block_device *bdev, block_t blkstart, block_t blklen)
1433{
1434 block_t lblkstart = blkstart;
1435
1436 if (!f2fs_bdev_support_discard(bdev))
1437 return 0;
1438
1439 trace_f2fs_queue_discard(bdev, blkstart, blklen);
1440
1441 if (f2fs_is_multi_device(sbi)) {
1442 int devi = f2fs_target_device_index(sbi, blkstart);
1443
1444 blkstart -= FDEV(devi).start_blk;
1445 }
1446 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1447 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
1448 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1449 return 0;
1450}
1451
1452static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
1453 struct discard_policy *dpolicy)
1454{
1455 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1456 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1457 struct rb_node **insert_p = NULL, *insert_parent = NULL;
1458 struct discard_cmd *dc;
1459 struct blk_plug plug;
1460 unsigned int pos = dcc->next_pos;
1461 unsigned int issued = 0;
1462 bool io_interrupted = false;
1463
1464 mutex_lock(&dcc->cmd_lock);
1465 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1466 NULL, pos,
1467 (struct rb_entry **)&prev_dc,
1468 (struct rb_entry **)&next_dc,
1469 &insert_p, &insert_parent, true, NULL);
1470 if (!dc)
1471 dc = next_dc;
1472
1473 blk_start_plug(&plug);
1474
1475 while (dc) {
1476 struct rb_node *node;
1477 int err = 0;
1478
1479 if (dc->state != D_PREP)
1480 goto next;
1481
1482 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
1483 io_interrupted = true;
1484 break;
1485 }
1486
1487 dcc->next_pos = dc->lstart + dc->len;
1488 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1489
1490 if (issued >= dpolicy->max_requests)
1491 break;
1492next:
1493 node = rb_next(&dc->rb_node);
1494 if (err)
1495 __remove_discard_cmd(sbi, dc);
1496 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1497 }
1498
1499 blk_finish_plug(&plug);
1500
1501 if (!dc)
1502 dcc->next_pos = 0;
1503
1504 mutex_unlock(&dcc->cmd_lock);
1505
1506 if (!issued && io_interrupted)
1507 issued = -1;
1508
1509 return issued;
1510}
1511static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1512 struct discard_policy *dpolicy);
1513
1514static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
1515 struct discard_policy *dpolicy)
1516{
1517 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1518 struct list_head *pend_list;
1519 struct discard_cmd *dc, *tmp;
1520 struct blk_plug plug;
1521 int i, issued;
1522 bool io_interrupted = false;
1523
1524 if (dpolicy->timeout)
1525 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);
1526
1527retry:
1528 issued = 0;
1529 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1530 if (dpolicy->timeout &&
1531 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1532 break;
1533
1534 if (i + 1 < dpolicy->granularity)
1535 break;
1536
1537 if (i < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered)
1538 return __issue_discard_cmd_orderly(sbi, dpolicy);
1539
1540 pend_list = &dcc->pend_list[i];
1541
1542 mutex_lock(&dcc->cmd_lock);
1543 if (list_empty(pend_list))
1544 goto next;
1545 if (unlikely(dcc->rbtree_check))
1546 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
1547 &dcc->root, false));
1548 blk_start_plug(&plug);
1549 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1550 f2fs_bug_on(sbi, dc->state != D_PREP);
1551
1552 if (dpolicy->timeout &&
1553 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1554 break;
1555
1556 if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
1557 !is_idle(sbi, DISCARD_TIME)) {
1558 io_interrupted = true;
1559 break;
1560 }
1561
1562 __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1563
1564 if (issued >= dpolicy->max_requests)
1565 break;
1566 }
1567 blk_finish_plug(&plug);
1568next:
1569 mutex_unlock(&dcc->cmd_lock);
1570
1571 if (issued >= dpolicy->max_requests || io_interrupted)
1572 break;
1573 }
1574
1575 if (dpolicy->type == DPOLICY_UMOUNT && issued) {
1576 __wait_all_discard_cmd(sbi, dpolicy);
1577 goto retry;
1578 }
1579
1580 if (!issued && io_interrupted)
1581 issued = -1;
1582
1583 return issued;
1584}
1585
1586static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
1587{
1588 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1589 struct list_head *pend_list;
1590 struct discard_cmd *dc, *tmp;
1591 int i;
1592 bool dropped = false;
1593
1594 mutex_lock(&dcc->cmd_lock);
1595 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1596 pend_list = &dcc->pend_list[i];
1597 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1598 f2fs_bug_on(sbi, dc->state != D_PREP);
1599 __remove_discard_cmd(sbi, dc);
1600 dropped = true;
1601 }
1602 }
1603 mutex_unlock(&dcc->cmd_lock);
1604
1605 return dropped;
1606}
1607
1608void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
1609{
1610 __drop_discard_cmd(sbi);
1611}
1612
1613static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
1614 struct discard_cmd *dc)
1615{
1616 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1617 unsigned int len = 0;
1618
1619 wait_for_completion_io(&dc->wait);
1620 mutex_lock(&dcc->cmd_lock);
1621 f2fs_bug_on(sbi, dc->state != D_DONE);
1622 dc->ref--;
1623 if (!dc->ref) {
1624 if (!dc->error)
1625 len = dc->len;
1626 __remove_discard_cmd(sbi, dc);
1627 }
1628 mutex_unlock(&dcc->cmd_lock);
1629
1630 return len;
1631}
1632
1633static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
1634 struct discard_policy *dpolicy,
1635 block_t start, block_t end)
1636{
1637 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1638 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1639 &(dcc->fstrim_list) : &(dcc->wait_list);
1640 struct discard_cmd *dc, *tmp;
1641 bool need_wait;
1642 unsigned int trimmed = 0;
1643
1644next:
1645 need_wait = false;
1646
1647 mutex_lock(&dcc->cmd_lock);
1648 list_for_each_entry_safe(dc, tmp, wait_list, list) {
1649 if (dc->lstart + dc->len <= start || end <= dc->lstart)
1650 continue;
1651 if (dc->len < dpolicy->granularity)
1652 continue;
1653 if (dc->state == D_DONE && !dc->ref) {
1654 wait_for_completion_io(&dc->wait);
1655 if (!dc->error)
1656 trimmed += dc->len;
1657 __remove_discard_cmd(sbi, dc);
1658 } else {
1659 dc->ref++;
1660 need_wait = true;
1661 break;
1662 }
1663 }
1664 mutex_unlock(&dcc->cmd_lock);
1665
1666 if (need_wait) {
1667 trimmed += __wait_one_discard_bio(sbi, dc);
1668 goto next;
1669 }
1670
1671 return trimmed;
1672}
1673
1674static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1675 struct discard_policy *dpolicy)
1676{
1677 struct discard_policy dp;
1678 unsigned int discard_blks;
1679
1680 if (dpolicy)
1681 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
1682
1683
1684 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1);
1685 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1686 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1);
1687 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1688
1689 return discard_blks;
1690}
1691
1692
1693static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
1694{
1695 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1696 struct discard_cmd *dc;
1697 bool need_wait = false;
1698
1699 mutex_lock(&dcc->cmd_lock);
1700 dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root,
1701 NULL, blkaddr);
1702 if (dc) {
1703 if (dc->state == D_PREP) {
1704 __punch_discard_cmd(sbi, dc, blkaddr);
1705 } else {
1706 dc->ref++;
1707 need_wait = true;
1708 }
1709 }
1710 mutex_unlock(&dcc->cmd_lock);
1711
1712 if (need_wait)
1713 __wait_one_discard_bio(sbi, dc);
1714}
1715
1716void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
1717{
1718 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1719
1720 if (dcc && dcc->f2fs_issue_discard) {
1721 struct task_struct *discard_thread = dcc->f2fs_issue_discard;
1722
1723 dcc->f2fs_issue_discard = NULL;
1724 kthread_stop(discard_thread);
1725 }
1726}
1727
1728
1729bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
1730{
1731 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1732 struct discard_policy dpolicy;
1733 bool dropped;
1734
1735 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
1736 dcc->discard_granularity);
1737 __issue_discard_cmd(sbi, &dpolicy);
1738 dropped = __drop_discard_cmd(sbi);
1739
1740
1741 __wait_all_discard_cmd(sbi, NULL);
1742
1743 f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
1744 return dropped;
1745}
1746
1747static int issue_discard_thread(void *data)
1748{
1749 struct f2fs_sb_info *sbi = data;
1750 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1751 wait_queue_head_t *q = &dcc->discard_wait_queue;
1752 struct discard_policy dpolicy;
1753 unsigned int wait_ms = DEF_MIN_DISCARD_ISSUE_TIME;
1754 int issued;
1755
1756 set_freezable();
1757
1758 do {
1759 if (sbi->gc_mode == GC_URGENT_HIGH ||
1760 !f2fs_available_free_memory(sbi, DISCARD_CACHE))
1761 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);
1762 else
1763 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1764 dcc->discard_granularity);
1765
1766 if (!atomic_read(&dcc->discard_cmd_cnt))
1767 wait_ms = dpolicy.max_interval;
1768
1769 wait_event_interruptible_timeout(*q,
1770 kthread_should_stop() || freezing(current) ||
1771 dcc->discard_wake,
1772 msecs_to_jiffies(wait_ms));
1773
1774 if (dcc->discard_wake)
1775 dcc->discard_wake = 0;
1776
1777
1778 if (atomic_read(&dcc->queued_discard))
1779 __wait_all_discard_cmd(sbi, NULL);
1780
1781 if (try_to_freeze())
1782 continue;
1783 if (f2fs_readonly(sbi->sb))
1784 continue;
1785 if (kthread_should_stop())
1786 return 0;
1787 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
1788 wait_ms = dpolicy.max_interval;
1789 continue;
1790 }
1791 if (!atomic_read(&dcc->discard_cmd_cnt))
1792 continue;
1793
1794 sb_start_intwrite(sbi->sb);
1795
1796 issued = __issue_discard_cmd(sbi, &dpolicy);
1797 if (issued > 0) {
1798 __wait_all_discard_cmd(sbi, &dpolicy);
1799 wait_ms = dpolicy.min_interval;
1800 } else if (issued == -1) {
1801 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
1802 if (!wait_ms)
1803 wait_ms = dpolicy.mid_interval;
1804 } else {
1805 wait_ms = dpolicy.max_interval;
1806 }
1807
1808 sb_end_intwrite(sbi->sb);
1809
1810 } while (!kthread_should_stop());
1811 return 0;
1812}
1813
1814#ifdef CONFIG_BLK_DEV_ZONED
1815static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1816 struct block_device *bdev, block_t blkstart, block_t blklen)
1817{
1818 sector_t sector, nr_sects;
1819 block_t lblkstart = blkstart;
1820 int devi = 0;
1821
1822 if (f2fs_is_multi_device(sbi)) {
1823 devi = f2fs_target_device_index(sbi, blkstart);
1824 if (blkstart < FDEV(devi).start_blk ||
1825 blkstart > FDEV(devi).end_blk) {
1826 f2fs_err(sbi, "Invalid block %x", blkstart);
1827 return -EIO;
1828 }
1829 blkstart -= FDEV(devi).start_blk;
1830 }
1831
1832
1833 if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
1834 sector = SECTOR_FROM_BLOCK(blkstart);
1835 nr_sects = SECTOR_FROM_BLOCK(blklen);
1836
1837 if (sector & (bdev_zone_sectors(bdev) - 1) ||
1838 nr_sects != bdev_zone_sectors(bdev)) {
1839 f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
1840 devi, sbi->s_ndevs ? FDEV(devi).path : "",
1841 blkstart, blklen);
1842 return -EIO;
1843 }
1844 trace_f2fs_issue_reset_zone(bdev, blkstart);
1845 return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
1846 sector, nr_sects, GFP_NOFS);
1847 }
1848
1849
1850 return __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
1851}
1852#endif
1853
1854static int __issue_discard_async(struct f2fs_sb_info *sbi,
1855 struct block_device *bdev, block_t blkstart, block_t blklen)
1856{
1857#ifdef CONFIG_BLK_DEV_ZONED
1858 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
1859 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
1860#endif
1861 return __queue_discard_cmd(sbi, bdev, blkstart, blklen);
1862}
1863
1864static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
1865 block_t blkstart, block_t blklen)
1866{
1867 sector_t start = blkstart, len = 0;
1868 struct block_device *bdev;
1869 struct seg_entry *se;
1870 unsigned int offset;
1871 block_t i;
1872 int err = 0;
1873
1874 bdev = f2fs_target_device(sbi, blkstart, NULL);
1875
1876 for (i = blkstart; i < blkstart + blklen; i++, len++) {
1877 if (i != start) {
1878 struct block_device *bdev2 =
1879 f2fs_target_device(sbi, i, NULL);
1880
1881 if (bdev2 != bdev) {
1882 err = __issue_discard_async(sbi, bdev,
1883 start, len);
1884 if (err)
1885 return err;
1886 bdev = bdev2;
1887 start = i;
1888 len = 0;
1889 }
1890 }
1891
1892 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
1893 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
1894
1895 if (!f2fs_test_and_set_bit(offset, se->discard_map))
1896 sbi->discard_blks--;
1897 }
1898
1899 if (len)
1900 err = __issue_discard_async(sbi, bdev, start, len);
1901 return err;
1902}
1903
1904static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
1905 bool check_only)
1906{
1907 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1908 int max_blocks = sbi->blocks_per_seg;
1909 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
1910 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1911 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1912 unsigned long *discard_map = (unsigned long *)se->discard_map;
1913 unsigned long *dmap = SIT_I(sbi)->tmp_map;
1914 unsigned int start = 0, end = -1;
1915 bool force = (cpc->reason & CP_DISCARD);
1916 struct discard_entry *de = NULL;
1917 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
1918 int i;
1919
1920 if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi))
1921 return false;
1922
1923 if (!force) {
1924 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
1925 SM_I(sbi)->dcc_info->nr_discards >=
1926 SM_I(sbi)->dcc_info->max_discards)
1927 return false;
1928 }
1929
1930
1931 for (i = 0; i < entries; i++)
1932 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
1933 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
1934
1935 while (force || SM_I(sbi)->dcc_info->nr_discards <=
1936 SM_I(sbi)->dcc_info->max_discards) {
1937 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
1938 if (start >= max_blocks)
1939 break;
1940
1941 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
1942 if (force && start && end != max_blocks
1943 && (end - start) < cpc->trim_minlen)
1944 continue;
1945
1946 if (check_only)
1947 return true;
1948
1949 if (!de) {
1950 de = f2fs_kmem_cache_alloc(discard_entry_slab,
1951 GFP_F2FS_ZERO);
1952 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
1953 list_add_tail(&de->list, head);
1954 }
1955
1956 for (i = start; i < end; i++)
1957 __set_bit_le(i, (void *)de->discard_map);
1958
1959 SM_I(sbi)->dcc_info->nr_discards += end - start;
1960 }
1961 return false;
1962}
1963
1964static void release_discard_addr(struct discard_entry *entry)
1965{
1966 list_del(&entry->list);
1967 kmem_cache_free(discard_entry_slab, entry);
1968}
1969
1970void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
1971{
1972 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
1973 struct discard_entry *entry, *this;
1974
1975
1976 list_for_each_entry_safe(entry, this, head, list)
1977 release_discard_addr(entry);
1978}
1979
1980
1981
1982
1983static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
1984{
1985 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1986 unsigned int segno;
1987
1988 mutex_lock(&dirty_i->seglist_lock);
1989 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
1990 __set_test_and_free(sbi, segno, false);
1991 mutex_unlock(&dirty_i->seglist_lock);
1992}
1993
1994void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
1995 struct cp_control *cpc)
1996{
1997 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1998 struct list_head *head = &dcc->entry_list;
1999 struct discard_entry *entry, *this;
2000 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2001 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
2002 unsigned int start = 0, end = -1;
2003 unsigned int secno, start_segno;
2004 bool force = (cpc->reason & CP_DISCARD);
2005 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
2006
2007 mutex_lock(&dirty_i->seglist_lock);
2008
2009 while (1) {
2010 int i;
2011
2012 if (need_align && end != -1)
2013 end--;
2014 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
2015 if (start >= MAIN_SEGS(sbi))
2016 break;
2017 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
2018 start + 1);
2019
2020 if (need_align) {
2021 start = rounddown(start, sbi->segs_per_sec);
2022 end = roundup(end, sbi->segs_per_sec);
2023 }
2024
2025 for (i = start; i < end; i++) {
2026 if (test_and_clear_bit(i, prefree_map))
2027 dirty_i->nr_dirty[PRE]--;
2028 }
2029
2030 if (!f2fs_realtime_discard_enable(sbi))
2031 continue;
2032
2033 if (force && start >= cpc->trim_start &&
2034 (end - 1) <= cpc->trim_end)
2035 continue;
2036
2037 if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) {
2038 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
2039 (end - start) << sbi->log_blocks_per_seg);
2040 continue;
2041 }
2042next:
2043 secno = GET_SEC_FROM_SEG(sbi, start);
2044 start_segno = GET_SEG_FROM_SEC(sbi, secno);
2045 if (!IS_CURSEC(sbi, secno) &&
2046 !get_valid_blocks(sbi, start, true))
2047 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
2048 sbi->segs_per_sec << sbi->log_blocks_per_seg);
2049
2050 start = start_segno + sbi->segs_per_sec;
2051 if (start < end)
2052 goto next;
2053 else
2054 end = start - 1;
2055 }
2056 mutex_unlock(&dirty_i->seglist_lock);
2057
2058
2059 list_for_each_entry_safe(entry, this, head, list) {
2060 unsigned int cur_pos = 0, next_pos, len, total_len = 0;
2061 bool is_valid = test_bit_le(0, entry->discard_map);
2062
2063find_next:
2064 if (is_valid) {
2065 next_pos = find_next_zero_bit_le(entry->discard_map,
2066 sbi->blocks_per_seg, cur_pos);
2067 len = next_pos - cur_pos;
2068
2069 if (f2fs_sb_has_blkzoned(sbi) ||
2070 (force && len < cpc->trim_minlen))
2071 goto skip;
2072
2073 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
2074 len);
2075 total_len += len;
2076 } else {
2077 next_pos = find_next_bit_le(entry->discard_map,
2078 sbi->blocks_per_seg, cur_pos);
2079 }
2080skip:
2081 cur_pos = next_pos;
2082 is_valid = !is_valid;
2083
2084 if (cur_pos < sbi->blocks_per_seg)
2085 goto find_next;
2086
2087 release_discard_addr(entry);
2088 dcc->nr_discards -= total_len;
2089 }
2090
2091 wake_up_discard_thread(sbi, false);
2092}
2093
2094static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2095{
2096 dev_t dev = sbi->sb->s_bdev->bd_dev;
2097 struct discard_cmd_control *dcc;
2098 int err = 0, i;
2099
2100 if (SM_I(sbi)->dcc_info) {
2101 dcc = SM_I(sbi)->dcc_info;
2102 goto init_thread;
2103 }
2104
2105 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2106 if (!dcc)
2107 return -ENOMEM;
2108
2109 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2110 INIT_LIST_HEAD(&dcc->entry_list);
2111 for (i = 0; i < MAX_PLIST_NUM; i++)
2112 INIT_LIST_HEAD(&dcc->pend_list[i]);
2113 INIT_LIST_HEAD(&dcc->wait_list);
2114 INIT_LIST_HEAD(&dcc->fstrim_list);
2115 mutex_init(&dcc->cmd_lock);
2116 atomic_set(&dcc->issued_discard, 0);
2117 atomic_set(&dcc->queued_discard, 0);
2118 atomic_set(&dcc->discard_cmd_cnt, 0);
2119 dcc->nr_discards = 0;
2120 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
2121 dcc->undiscard_blks = 0;
2122 dcc->next_pos = 0;
2123 dcc->root = RB_ROOT_CACHED;
2124 dcc->rbtree_check = false;
2125
2126 init_waitqueue_head(&dcc->discard_wait_queue);
2127 SM_I(sbi)->dcc_info = dcc;
2128init_thread:
2129 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2130 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2131 if (IS_ERR(dcc->f2fs_issue_discard)) {
2132 err = PTR_ERR(dcc->f2fs_issue_discard);
2133 kfree(dcc);
2134 SM_I(sbi)->dcc_info = NULL;
2135 return err;
2136 }
2137
2138 return err;
2139}
2140
2141static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2142{
2143 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2144
2145 if (!dcc)
2146 return;
2147
2148 f2fs_stop_discard_thread(sbi);
2149
2150
2151
2152
2153
2154 if (unlikely(atomic_read(&dcc->discard_cmd_cnt)))
2155 f2fs_issue_discard_timeout(sbi);
2156
2157 kfree(dcc);
2158 SM_I(sbi)->dcc_info = NULL;
2159}
2160
2161static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
2162{
2163 struct sit_info *sit_i = SIT_I(sbi);
2164
2165 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
2166 sit_i->dirty_sentries++;
2167 return false;
2168 }
2169
2170 return true;
2171}
2172
2173static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
2174 unsigned int segno, int modified)
2175{
2176 struct seg_entry *se = get_seg_entry(sbi, segno);
2177
2178 se->type = type;
2179 if (modified)
2180 __mark_sit_entry_dirty(sbi, segno);
2181}
2182
2183static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
2184 block_t blkaddr)
2185{
2186 unsigned int segno = GET_SEGNO(sbi, blkaddr);
2187
2188 if (segno == NULL_SEGNO)
2189 return 0;
2190 return get_seg_entry(sbi, segno)->mtime;
2191}
2192
2193static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
2194 unsigned long long old_mtime)
2195{
2196 struct seg_entry *se;
2197 unsigned int segno = GET_SEGNO(sbi, blkaddr);
2198 unsigned long long ctime = get_mtime(sbi, false);
2199 unsigned long long mtime = old_mtime ? old_mtime : ctime;
2200
2201 if (segno == NULL_SEGNO)
2202 return;
2203
2204 se = get_seg_entry(sbi, segno);
2205
2206 if (!se->mtime)
2207 se->mtime = mtime;
2208 else
2209 se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
2210 se->valid_blocks + 1);
2211
2212 if (ctime > SIT_I(sbi)->max_mtime)
2213 SIT_I(sbi)->max_mtime = ctime;
2214}
2215
2216static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
2217{
2218 struct seg_entry *se;
2219 unsigned int segno, offset;
2220 long int new_vblocks;
2221 bool exist;
2222#ifdef CONFIG_F2FS_CHECK_FS
2223 bool mir_exist;
2224#endif
2225
2226 segno = GET_SEGNO(sbi, blkaddr);
2227
2228 se = get_seg_entry(sbi, segno);
2229 new_vblocks = se->valid_blocks + del;
2230 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2231
2232 f2fs_bug_on(sbi, (new_vblocks < 0 ||
2233 (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
2234
2235 se->valid_blocks = new_vblocks;
2236
2237
2238 if (del > 0) {
2239 exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
2240#ifdef CONFIG_F2FS_CHECK_FS
2241 mir_exist = f2fs_test_and_set_bit(offset,
2242 se->cur_valid_map_mir);
2243 if (unlikely(exist != mir_exist)) {
2244 f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
2245 blkaddr, exist);
2246 f2fs_bug_on(sbi, 1);
2247 }
2248#endif
2249 if (unlikely(exist)) {
2250 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
2251 blkaddr);
2252 f2fs_bug_on(sbi, 1);
2253 se->valid_blocks--;
2254 del = 0;
2255 }
2256
2257 if (!f2fs_test_and_set_bit(offset, se->discard_map))
2258 sbi->discard_blks--;
2259
2260
2261
2262
2263
2264 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2265 if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
2266 se->ckpt_valid_blocks++;
2267 }
2268 } else {
2269 exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);
2270#ifdef CONFIG_F2FS_CHECK_FS
2271 mir_exist = f2fs_test_and_clear_bit(offset,
2272 se->cur_valid_map_mir);
2273 if (unlikely(exist != mir_exist)) {
2274 f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
2275 blkaddr, exist);
2276 f2fs_bug_on(sbi, 1);
2277 }
2278#endif
2279 if (unlikely(!exist)) {
2280 f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",
2281 blkaddr);
2282 f2fs_bug_on(sbi, 1);
2283 se->valid_blocks++;
2284 del = 0;
2285 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2286
2287
2288
2289
2290
2291
2292 if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
2293 spin_lock(&sbi->stat_lock);
2294 sbi->unusable_block_count++;
2295 spin_unlock(&sbi->stat_lock);
2296 }
2297 }
2298
2299 if (f2fs_test_and_clear_bit(offset, se->discard_map))
2300 sbi->discard_blks++;
2301 }
2302 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2303 se->ckpt_valid_blocks += del;
2304
2305 __mark_sit_entry_dirty(sbi, segno);
2306
2307
2308 SIT_I(sbi)->written_valid_blocks += del;
2309
2310 if (__is_large_section(sbi))
2311 get_sec_entry(sbi, segno)->valid_blocks += del;
2312}
2313
2314void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
2315{
2316 unsigned int segno = GET_SEGNO(sbi, addr);
2317 struct sit_info *sit_i = SIT_I(sbi);
2318
2319 f2fs_bug_on(sbi, addr == NULL_ADDR);
2320 if (addr == NEW_ADDR || addr == COMPRESS_ADDR)
2321 return;
2322
2323 invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);
2324
2325
2326 down_write(&sit_i->sentry_lock);
2327
2328 update_segment_mtime(sbi, addr, 0);
2329 update_sit_entry(sbi, addr, -1);
2330
2331
2332 locate_dirty_segment(sbi, segno);
2333
2334 up_write(&sit_i->sentry_lock);
2335}
2336
2337bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
2338{
2339 struct sit_info *sit_i = SIT_I(sbi);
2340 unsigned int segno, offset;
2341 struct seg_entry *se;
2342 bool is_cp = false;
2343
2344 if (!__is_valid_data_blkaddr(blkaddr))
2345 return true;
2346
2347 down_read(&sit_i->sentry_lock);
2348
2349 segno = GET_SEGNO(sbi, blkaddr);
2350 se = get_seg_entry(sbi, segno);
2351 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2352
2353 if (f2fs_test_bit(offset, se->ckpt_valid_map))
2354 is_cp = true;
2355
2356 up_read(&sit_i->sentry_lock);
2357
2358 return is_cp;
2359}
2360
2361
2362
2363
2364static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
2365 struct f2fs_summary *sum)
2366{
2367 struct curseg_info *curseg = CURSEG_I(sbi, type);
2368 void *addr = curseg->sum_blk;
2369
2370 addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
2371 memcpy(addr, sum, sizeof(struct f2fs_summary));
2372}
2373
2374
2375
2376
2377int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
2378{
2379 int valid_sum_count = 0;
2380 int i, sum_in_page;
2381
2382 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2383 if (sbi->ckpt->alloc_type[i] == SSR)
2384 valid_sum_count += sbi->blocks_per_seg;
2385 else {
2386 if (for_ra)
2387 valid_sum_count += le16_to_cpu(
2388 F2FS_CKPT(sbi)->cur_data_blkoff[i]);
2389 else
2390 valid_sum_count += curseg_blkoff(sbi, i);
2391 }
2392 }
2393
2394 sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
2395 SUM_FOOTER_SIZE) / SUMMARY_SIZE;
2396 if (valid_sum_count <= sum_in_page)
2397 return 1;
2398 else if ((valid_sum_count - sum_in_page) <=
2399 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
2400 return 2;
2401 return 3;
2402}
2403
2404
2405
2406
2407struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
2408{
2409 if (unlikely(f2fs_cp_error(sbi)))
2410 return ERR_PTR(-EIO);
2411 return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));
2412}
2413
2414void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
2415 void *src, block_t blk_addr)
2416{
2417 struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2418
2419 memcpy(page_address(page), src, PAGE_SIZE);
2420 set_page_dirty(page);
2421 f2fs_put_page(page, 1);
2422}
2423
2424static void write_sum_page(struct f2fs_sb_info *sbi,
2425 struct f2fs_summary_block *sum_blk, block_t blk_addr)
2426{
2427 f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
2428}
2429
2430static void write_current_sum_page(struct f2fs_sb_info *sbi,
2431 int type, block_t blk_addr)
2432{
2433 struct curseg_info *curseg = CURSEG_I(sbi, type);
2434 struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2435 struct f2fs_summary_block *src = curseg->sum_blk;
2436 struct f2fs_summary_block *dst;
2437
2438 dst = (struct f2fs_summary_block *)page_address(page);
2439 memset(dst, 0, PAGE_SIZE);
2440
2441 mutex_lock(&curseg->curseg_mutex);
2442
2443 down_read(&curseg->journal_rwsem);
2444 memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
2445 up_read(&curseg->journal_rwsem);
2446
2447 memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
2448 memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
2449
2450 mutex_unlock(&curseg->curseg_mutex);
2451
2452 set_page_dirty(page);
2453 f2fs_put_page(page, 1);
2454}
2455
2456static int is_next_segment_free(struct f2fs_sb_info *sbi,
2457 struct curseg_info *curseg, int type)
2458{
2459 unsigned int segno = curseg->segno + 1;
2460 struct free_segmap_info *free_i = FREE_I(sbi);
2461
2462 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
2463 return !test_bit(segno, free_i->free_segmap);
2464 return 0;
2465}
2466
2467
2468
2469
2470
2471static void get_new_segment(struct f2fs_sb_info *sbi,
2472 unsigned int *newseg, bool new_sec, int dir)
2473{
2474 struct free_segmap_info *free_i = FREE_I(sbi);
2475 unsigned int segno, secno, zoneno;
2476 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
2477 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
2478 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
2479 unsigned int left_start = hint;
2480 bool init = true;
2481 int go_left = 0;
2482 int i;
2483
2484 spin_lock(&free_i->segmap_lock);
2485
2486 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
2487 segno = find_next_zero_bit(free_i->free_segmap,
2488 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
2489 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
2490 goto got_it;
2491 }
2492find_other_zone:
2493 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
2494 if (secno >= MAIN_SECS(sbi)) {
2495 if (dir == ALLOC_RIGHT) {
2496 secno = find_next_zero_bit(free_i->free_secmap,
2497 MAIN_SECS(sbi), 0);
2498 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
2499 } else {
2500 go_left = 1;
2501 left_start = hint - 1;
2502 }
2503 }
2504 if (go_left == 0)
2505 goto skip_left;
2506
2507 while (test_bit(left_start, free_i->free_secmap)) {
2508 if (left_start > 0) {
2509 left_start--;
2510 continue;
2511 }
2512 left_start = find_next_zero_bit(free_i->free_secmap,
2513 MAIN_SECS(sbi), 0);
2514 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
2515 break;
2516 }
2517 secno = left_start;
2518skip_left:
2519 segno = GET_SEG_FROM_SEC(sbi, secno);
2520 zoneno = GET_ZONE_FROM_SEC(sbi, secno);
2521
2522
2523 if (!init)
2524 goto got_it;
2525 if (sbi->secs_per_zone == 1)
2526 goto got_it;
2527 if (zoneno == old_zoneno)
2528 goto got_it;
2529 if (dir == ALLOC_LEFT) {
2530 if (!go_left && zoneno + 1 >= total_zones)
2531 goto got_it;
2532 if (go_left && zoneno == 0)
2533 goto got_it;
2534 }
2535 for (i = 0; i < NR_CURSEG_TYPE; i++)
2536 if (CURSEG_I(sbi, i)->zone == zoneno)
2537 break;
2538
2539 if (i < NR_CURSEG_TYPE) {
2540
2541 if (go_left)
2542 hint = zoneno * sbi->secs_per_zone - 1;
2543 else if (zoneno + 1 >= total_zones)
2544 hint = 0;
2545 else
2546 hint = (zoneno + 1) * sbi->secs_per_zone;
2547 init = false;
2548 goto find_other_zone;
2549 }
2550got_it:
2551
2552 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
2553 __set_inuse(sbi, segno);
2554 *newseg = segno;
2555 spin_unlock(&free_i->segmap_lock);
2556}
2557
2558static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
2559{
2560 struct curseg_info *curseg = CURSEG_I(sbi, type);
2561 struct summary_footer *sum_footer;
2562 unsigned short seg_type = curseg->seg_type;
2563
2564 curseg->inited = true;
2565 curseg->segno = curseg->next_segno;
2566 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
2567 curseg->next_blkoff = 0;
2568 curseg->next_segno = NULL_SEGNO;
2569
2570 sum_footer = &(curseg->sum_blk->footer);
2571 memset(sum_footer, 0, sizeof(struct summary_footer));
2572
2573 sanity_check_seg_type(sbi, seg_type);
2574
2575 if (IS_DATASEG(seg_type))
2576 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
2577 if (IS_NODESEG(seg_type))
2578 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
2579 __set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
2580}
2581
2582static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
2583{
2584 struct curseg_info *curseg = CURSEG_I(sbi, type);
2585 unsigned short seg_type = curseg->seg_type;
2586
2587 sanity_check_seg_type(sbi, seg_type);
2588
2589
2590 if (__is_large_section(sbi))
2591 return curseg->segno;
2592
2593
2594 if (!curseg->inited)
2595 return 0;
2596
2597 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2598 return 0;
2599
2600 if (test_opt(sbi, NOHEAP) &&
2601 (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type)))
2602 return 0;
2603
2604 if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
2605 return SIT_I(sbi)->last_victim[ALLOC_NEXT];
2606
2607
2608 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
2609 return 0;
2610
2611 return curseg->segno;
2612}
2613
2614
2615
2616
2617
2618static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
2619{
2620 struct curseg_info *curseg = CURSEG_I(sbi, type);
2621 unsigned short seg_type = curseg->seg_type;
2622 unsigned int segno = curseg->segno;
2623 int dir = ALLOC_LEFT;
2624
2625 if (curseg->inited)
2626 write_sum_page(sbi, curseg->sum_blk,
2627 GET_SUM_BLOCK(sbi, segno));
2628 if (seg_type == CURSEG_WARM_DATA || seg_type == CURSEG_COLD_DATA)
2629 dir = ALLOC_RIGHT;
2630
2631 if (test_opt(sbi, NOHEAP))
2632 dir = ALLOC_RIGHT;
2633
2634 segno = __get_next_segno(sbi, type);
2635 get_new_segment(sbi, &segno, new_sec, dir);
2636 curseg->next_segno = segno;
2637 reset_curseg(sbi, type, 1);
2638 curseg->alloc_type = LFS;
2639}
2640
2641static int __next_free_blkoff(struct f2fs_sb_info *sbi,
2642 int segno, block_t start)
2643{
2644 struct seg_entry *se = get_seg_entry(sbi, segno);
2645 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2646 unsigned long *target_map = SIT_I(sbi)->tmp_map;
2647 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2648 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2649 int i;
2650
2651 for (i = 0; i < entries; i++)
2652 target_map[i] = ckpt_map[i] | cur_map[i];
2653
2654 return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
2655}
2656
2657
2658
2659
2660
2661
2662static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
2663 struct curseg_info *seg)
2664{
2665 if (seg->alloc_type == SSR)
2666 seg->next_blkoff =
2667 __next_free_blkoff(sbi, seg->segno,
2668 seg->next_blkoff + 1);
2669 else
2670 seg->next_blkoff++;
2671}
2672
2673bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
2674{
2675 return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg;
2676}
2677
2678
2679
2680
2681
2682static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush)
2683{
2684 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2685 struct curseg_info *curseg = CURSEG_I(sbi, type);
2686 unsigned int new_segno = curseg->next_segno;
2687 struct f2fs_summary_block *sum_node;
2688 struct page *sum_page;
2689
2690 if (flush)
2691 write_sum_page(sbi, curseg->sum_blk,
2692 GET_SUM_BLOCK(sbi, curseg->segno));
2693
2694 __set_test_and_inuse(sbi, new_segno);
2695
2696 mutex_lock(&dirty_i->seglist_lock);
2697 __remove_dirty_segment(sbi, new_segno, PRE);
2698 __remove_dirty_segment(sbi, new_segno, DIRTY);
2699 mutex_unlock(&dirty_i->seglist_lock);
2700
2701 reset_curseg(sbi, type, 1);
2702 curseg->alloc_type = SSR;
2703 curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
2704
2705 sum_page = f2fs_get_sum_page(sbi, new_segno);
2706 if (IS_ERR(sum_page)) {
2707
2708 memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE);
2709 return;
2710 }
2711 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
2712 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
2713 f2fs_put_page(sum_page, 1);
2714}
2715
2716static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2717 int alloc_mode, unsigned long long age);
2718
2719static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,
2720 int target_type, int alloc_mode,
2721 unsigned long long age)
2722{
2723 struct curseg_info *curseg = CURSEG_I(sbi, type);
2724
2725 curseg->seg_type = target_type;
2726
2727 if (get_ssr_segment(sbi, type, alloc_mode, age)) {
2728 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
2729
2730 curseg->seg_type = se->type;
2731 change_curseg(sbi, type, true);
2732 } else {
2733
2734 curseg->seg_type = CURSEG_COLD_DATA;
2735 new_curseg(sbi, type, true);
2736 }
2737 stat_inc_seg_type(sbi, curseg);
2738}
2739
2740static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)
2741{
2742 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
2743
2744 if (!sbi->am.atgc_enabled)
2745 return;
2746
2747 down_read(&SM_I(sbi)->curseg_lock);
2748
2749 mutex_lock(&curseg->curseg_mutex);
2750 down_write(&SIT_I(sbi)->sentry_lock);
2751
2752 get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);
2753
2754 up_write(&SIT_I(sbi)->sentry_lock);
2755 mutex_unlock(&curseg->curseg_mutex);
2756
2757 up_read(&SM_I(sbi)->curseg_lock);
2758
2759}
2760void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
2761{
2762 __f2fs_init_atgc_curseg(sbi);
2763}
2764
2765static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2766{
2767 struct curseg_info *curseg = CURSEG_I(sbi, type);
2768
2769 mutex_lock(&curseg->curseg_mutex);
2770 if (!curseg->inited)
2771 goto out;
2772
2773 if (get_valid_blocks(sbi, curseg->segno, false)) {
2774 write_sum_page(sbi, curseg->sum_blk,
2775 GET_SUM_BLOCK(sbi, curseg->segno));
2776 } else {
2777 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2778 __set_test_and_free(sbi, curseg->segno, true);
2779 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2780 }
2781out:
2782 mutex_unlock(&curseg->curseg_mutex);
2783}
2784
2785void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
2786{
2787 __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2788
2789 if (sbi->am.atgc_enabled)
2790 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2791}
2792
2793static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2794{
2795 struct curseg_info *curseg = CURSEG_I(sbi, type);
2796
2797 mutex_lock(&curseg->curseg_mutex);
2798 if (!curseg->inited)
2799 goto out;
2800 if (get_valid_blocks(sbi, curseg->segno, false))
2801 goto out;
2802
2803 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2804 __set_test_and_inuse(sbi, curseg->segno);
2805 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2806out:
2807 mutex_unlock(&curseg->curseg_mutex);
2808}
2809
2810void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
2811{
2812 __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2813
2814 if (sbi->am.atgc_enabled)
2815 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2816}
2817
2818static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2819 int alloc_mode, unsigned long long age)
2820{
2821 struct curseg_info *curseg = CURSEG_I(sbi, type);
2822 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
2823 unsigned segno = NULL_SEGNO;
2824 unsigned short seg_type = curseg->seg_type;
2825 int i, cnt;
2826 bool reversed = false;
2827
2828 sanity_check_seg_type(sbi, seg_type);
2829
2830
2831 if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {
2832 curseg->next_segno = segno;
2833 return 1;
2834 }
2835
2836
2837 if (IS_NODESEG(seg_type)) {
2838 if (seg_type >= CURSEG_WARM_NODE) {
2839 reversed = true;
2840 i = CURSEG_COLD_NODE;
2841 } else {
2842 i = CURSEG_HOT_NODE;
2843 }
2844 cnt = NR_CURSEG_NODE_TYPE;
2845 } else {
2846 if (seg_type >= CURSEG_WARM_DATA) {
2847 reversed = true;
2848 i = CURSEG_COLD_DATA;
2849 } else {
2850 i = CURSEG_HOT_DATA;
2851 }
2852 cnt = NR_CURSEG_DATA_TYPE;
2853 }
2854
2855 for (; cnt-- > 0; reversed ? i-- : i++) {
2856 if (i == seg_type)
2857 continue;
2858 if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {
2859 curseg->next_segno = segno;
2860 return 1;
2861 }
2862 }
2863
2864
2865 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2866 segno = get_free_segment(sbi);
2867 if (segno != NULL_SEGNO) {
2868 curseg->next_segno = segno;
2869 return 1;
2870 }
2871 }
2872 return 0;
2873}
2874
2875
2876
2877
2878
2879static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
2880 int type, bool force)
2881{
2882 struct curseg_info *curseg = CURSEG_I(sbi, type);
2883
2884 if (force)
2885 new_curseg(sbi, type, true);
2886 else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
2887 curseg->seg_type == CURSEG_WARM_NODE)
2888 new_curseg(sbi, type, false);
2889 else if (curseg->alloc_type == LFS &&
2890 is_next_segment_free(sbi, curseg, type) &&
2891 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2892 new_curseg(sbi, type, false);
2893 else if (f2fs_need_SSR(sbi) &&
2894 get_ssr_segment(sbi, type, SSR, 0))
2895 change_curseg(sbi, type, true);
2896 else
2897 new_curseg(sbi, type, false);
2898
2899 stat_inc_seg_type(sbi, curseg);
2900}
2901
2902void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
2903 unsigned int start, unsigned int end)
2904{
2905 struct curseg_info *curseg = CURSEG_I(sbi, type);
2906 unsigned int segno;
2907
2908 down_read(&SM_I(sbi)->curseg_lock);
2909 mutex_lock(&curseg->curseg_mutex);
2910 down_write(&SIT_I(sbi)->sentry_lock);
2911
2912 segno = CURSEG_I(sbi, type)->segno;
2913 if (segno < start || segno > end)
2914 goto unlock;
2915
2916 if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
2917 change_curseg(sbi, type, true);
2918 else
2919 new_curseg(sbi, type, true);
2920
2921 stat_inc_seg_type(sbi, curseg);
2922
2923 locate_dirty_segment(sbi, segno);
2924unlock:
2925 up_write(&SIT_I(sbi)->sentry_lock);
2926
2927 if (segno != curseg->segno)
2928 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
2929 type, segno, curseg->segno);
2930
2931 mutex_unlock(&curseg->curseg_mutex);
2932 up_read(&SM_I(sbi)->curseg_lock);
2933}
2934
2935static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
2936 bool new_sec, bool force)
2937{
2938 struct curseg_info *curseg = CURSEG_I(sbi, type);
2939 unsigned int old_segno;
2940
2941 if (!curseg->inited)
2942 goto alloc;
2943
2944 if (force || curseg->next_blkoff ||
2945 get_valid_blocks(sbi, curseg->segno, new_sec))
2946 goto alloc;
2947
2948 if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
2949 return;
2950alloc:
2951 old_segno = curseg->segno;
2952 SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
2953 locate_dirty_segment(sbi, old_segno);
2954}
2955
2956static void __allocate_new_section(struct f2fs_sb_info *sbi,
2957 int type, bool force)
2958{
2959 __allocate_new_segment(sbi, type, true, force);
2960}
2961
2962void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
2963{
2964 down_read(&SM_I(sbi)->curseg_lock);
2965 down_write(&SIT_I(sbi)->sentry_lock);
2966 __allocate_new_section(sbi, type, force);
2967 up_write(&SIT_I(sbi)->sentry_lock);
2968 up_read(&SM_I(sbi)->curseg_lock);
2969}
2970
2971void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
2972{
2973 int i;
2974
2975 down_read(&SM_I(sbi)->curseg_lock);
2976 down_write(&SIT_I(sbi)->sentry_lock);
2977 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
2978 __allocate_new_segment(sbi, i, false, false);
2979 up_write(&SIT_I(sbi)->sentry_lock);
2980 up_read(&SM_I(sbi)->curseg_lock);
2981}
2982
2983static const struct segment_allocation default_salloc_ops = {
2984 .allocate_segment = allocate_segment_by_default,
2985};
2986
2987bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
2988 struct cp_control *cpc)
2989{
2990 __u64 trim_start = cpc->trim_start;
2991 bool has_candidate = false;
2992
2993 down_write(&SIT_I(sbi)->sentry_lock);
2994 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
2995 if (add_discard_addrs(sbi, cpc, true)) {
2996 has_candidate = true;
2997 break;
2998 }
2999 }
3000 up_write(&SIT_I(sbi)->sentry_lock);
3001
3002 cpc->trim_start = trim_start;
3003 return has_candidate;
3004}
3005
3006static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
3007 struct discard_policy *dpolicy,
3008 unsigned int start, unsigned int end)
3009{
3010 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
3011 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
3012 struct rb_node **insert_p = NULL, *insert_parent = NULL;
3013 struct discard_cmd *dc;
3014 struct blk_plug plug;
3015 int issued;
3016 unsigned int trimmed = 0;
3017
3018next:
3019 issued = 0;
3020
3021 mutex_lock(&dcc->cmd_lock);
3022 if (unlikely(dcc->rbtree_check))
3023 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
3024 &dcc->root, false));
3025
3026 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
3027 NULL, start,
3028 (struct rb_entry **)&prev_dc,
3029 (struct rb_entry **)&next_dc,
3030 &insert_p, &insert_parent, true, NULL);
3031 if (!dc)
3032 dc = next_dc;
3033
3034 blk_start_plug(&plug);
3035
3036 while (dc && dc->lstart <= end) {
3037 struct rb_node *node;
3038 int err = 0;
3039
3040 if (dc->len < dpolicy->granularity)
3041 goto skip;
3042
3043 if (dc->state != D_PREP) {
3044 list_move_tail(&dc->list, &dcc->fstrim_list);
3045 goto skip;
3046 }
3047
3048 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
3049
3050 if (issued >= dpolicy->max_requests) {
3051 start = dc->lstart + dc->len;
3052
3053 if (err)
3054 __remove_discard_cmd(sbi, dc);
3055
3056 blk_finish_plug(&plug);
3057 mutex_unlock(&dcc->cmd_lock);
3058 trimmed += __wait_all_discard_cmd(sbi, NULL);
3059 congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
3060 goto next;
3061 }
3062skip:
3063 node = rb_next(&dc->rb_node);
3064 if (err)
3065 __remove_discard_cmd(sbi, dc);
3066 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
3067
3068 if (fatal_signal_pending(current))
3069 break;
3070 }
3071
3072 blk_finish_plug(&plug);
3073 mutex_unlock(&dcc->cmd_lock);
3074
3075 return trimmed;
3076}
3077
3078int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
3079{
3080 __u64 start = F2FS_BYTES_TO_BLK(range->start);
3081 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
3082 unsigned int start_segno, end_segno;
3083 block_t start_block, end_block;
3084 struct cp_control cpc;
3085 struct discard_policy dpolicy;
3086 unsigned long long trimmed = 0;
3087 int err = 0;
3088 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
3089
3090 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
3091 return -EINVAL;
3092
3093 if (end < MAIN_BLKADDR(sbi))
3094 goto out;
3095
3096 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
3097 f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
3098 return -EFSCORRUPTED;
3099 }
3100
3101
3102 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
3103 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
3104 GET_SEGNO(sbi, end);
3105 if (need_align) {
3106 start_segno = rounddown(start_segno, sbi->segs_per_sec);
3107 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;
3108 }
3109
3110 cpc.reason = CP_DISCARD;
3111 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
3112 cpc.trim_start = start_segno;
3113 cpc.trim_end = end_segno;
3114
3115 if (sbi->discard_blks == 0)
3116 goto out;
3117
3118 down_write(&sbi->gc_lock);
3119 err = f2fs_write_checkpoint(sbi, &cpc);
3120 up_write(&sbi->gc_lock);
3121 if (err)
3122 goto out;
3123
3124
3125
3126
3127
3128
3129
3130 if (f2fs_realtime_discard_enable(sbi))
3131 goto out;
3132
3133 start_block = START_BLOCK(sbi, start_segno);
3134 end_block = START_BLOCK(sbi, end_segno + 1);
3135
3136 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
3137 trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
3138 start_block, end_block);
3139
3140 trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
3141 start_block, end_block);
3142out:
3143 if (!err)
3144 range->len = F2FS_BLK_TO_BYTES(trimmed);
3145 return err;
3146}
3147
3148static bool __has_curseg_space(struct f2fs_sb_info *sbi,
3149 struct curseg_info *curseg)
3150{
3151 return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi,
3152 curseg->segno);
3153}
3154
3155int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
3156{
3157 switch (hint) {
3158 case WRITE_LIFE_SHORT:
3159 return CURSEG_HOT_DATA;
3160 case WRITE_LIFE_EXTREME:
3161 return CURSEG_COLD_DATA;
3162 default:
3163 return CURSEG_WARM_DATA;
3164 }
3165}
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
3229 enum page_type type, enum temp_type temp)
3230{
3231 if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) {
3232 if (type == DATA) {
3233 if (temp == WARM)
3234 return WRITE_LIFE_NOT_SET;
3235 else if (temp == HOT)
3236 return WRITE_LIFE_SHORT;
3237 else if (temp == COLD)
3238 return WRITE_LIFE_EXTREME;
3239 } else {
3240 return WRITE_LIFE_NOT_SET;
3241 }
3242 } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) {
3243 if (type == DATA) {
3244 if (temp == WARM)
3245 return WRITE_LIFE_LONG;
3246 else if (temp == HOT)
3247 return WRITE_LIFE_SHORT;
3248 else if (temp == COLD)
3249 return WRITE_LIFE_EXTREME;
3250 } else if (type == NODE) {
3251 if (temp == WARM || temp == HOT)
3252 return WRITE_LIFE_NOT_SET;
3253 else if (temp == COLD)
3254 return WRITE_LIFE_NONE;
3255 } else if (type == META) {
3256 return WRITE_LIFE_MEDIUM;
3257 }
3258 }
3259 return WRITE_LIFE_NOT_SET;
3260}
3261
3262static int __get_segment_type_2(struct f2fs_io_info *fio)
3263{
3264 if (fio->type == DATA)
3265 return CURSEG_HOT_DATA;
3266 else
3267 return CURSEG_HOT_NODE;
3268}
3269
3270static int __get_segment_type_4(struct f2fs_io_info *fio)
3271{
3272 if (fio->type == DATA) {
3273 struct inode *inode = fio->page->mapping->host;
3274
3275 if (S_ISDIR(inode->i_mode))
3276 return CURSEG_HOT_DATA;
3277 else
3278 return CURSEG_COLD_DATA;
3279 } else {
3280 if (IS_DNODE(fio->page) && is_cold_node(fio->page))
3281 return CURSEG_WARM_NODE;
3282 else
3283 return CURSEG_COLD_NODE;
3284 }
3285}
3286
3287static int __get_segment_type_6(struct f2fs_io_info *fio)
3288{
3289 if (fio->type == DATA) {
3290 struct inode *inode = fio->page->mapping->host;
3291
3292 if (is_cold_data(fio->page)) {
3293 if (fio->sbi->am.atgc_enabled &&
3294 (fio->io_type == FS_DATA_IO) &&
3295 (fio->sbi->gc_mode != GC_URGENT_HIGH))
3296 return CURSEG_ALL_DATA_ATGC;
3297 else
3298 return CURSEG_COLD_DATA;
3299 }
3300 if (file_is_cold(inode) || f2fs_need_compress_data(inode))
3301 return CURSEG_COLD_DATA;
3302 if (file_is_hot(inode) ||
3303 is_inode_flag_set(inode, FI_HOT_DATA) ||
3304 f2fs_is_atomic_file(inode) ||
3305 f2fs_is_volatile_file(inode))
3306 return CURSEG_HOT_DATA;
3307 return f2fs_rw_hint_to_seg_type(inode->i_write_hint);
3308 } else {
3309 if (IS_DNODE(fio->page))
3310 return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
3311 CURSEG_HOT_NODE;
3312 return CURSEG_COLD_NODE;
3313 }
3314}
3315
3316static int __get_segment_type(struct f2fs_io_info *fio)
3317{
3318 int type = 0;
3319
3320 switch (F2FS_OPTION(fio->sbi).active_logs) {
3321 case 2:
3322 type = __get_segment_type_2(fio);
3323 break;
3324 case 4:
3325 type = __get_segment_type_4(fio);
3326 break;
3327 case 6:
3328 type = __get_segment_type_6(fio);
3329 break;
3330 default:
3331 f2fs_bug_on(fio->sbi, true);
3332 }
3333
3334 if (IS_HOT(type))
3335 fio->temp = HOT;
3336 else if (IS_WARM(type))
3337 fio->temp = WARM;
3338 else
3339 fio->temp = COLD;
3340 return type;
3341}
3342
3343void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3344 block_t old_blkaddr, block_t *new_blkaddr,
3345 struct f2fs_summary *sum, int type,
3346 struct f2fs_io_info *fio)
3347{
3348 struct sit_info *sit_i = SIT_I(sbi);
3349 struct curseg_info *curseg = CURSEG_I(sbi, type);
3350 unsigned long long old_mtime;
3351 bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
3352 struct seg_entry *se = NULL;
3353
3354 down_read(&SM_I(sbi)->curseg_lock);
3355
3356 mutex_lock(&curseg->curseg_mutex);
3357 down_write(&sit_i->sentry_lock);
3358
3359 if (from_gc) {
3360 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
3361 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
3362 sanity_check_seg_type(sbi, se->type);
3363 f2fs_bug_on(sbi, IS_NODESEG(se->type));
3364 }
3365 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
3366
3367 f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);
3368
3369 f2fs_wait_discard_bio(sbi, *new_blkaddr);
3370
3371
3372
3373
3374
3375
3376 __add_sum_entry(sbi, type, sum);
3377
3378 __refresh_next_blkoff(sbi, curseg);
3379
3380 stat_inc_block_count(sbi, curseg);
3381
3382 if (from_gc) {
3383 old_mtime = get_segment_mtime(sbi, old_blkaddr);
3384 } else {
3385 update_segment_mtime(sbi, old_blkaddr, 0);
3386 old_mtime = 0;
3387 }
3388 update_segment_mtime(sbi, *new_blkaddr, old_mtime);
3389
3390
3391
3392
3393
3394 update_sit_entry(sbi, *new_blkaddr, 1);
3395 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
3396 update_sit_entry(sbi, old_blkaddr, -1);
3397
3398 if (!__has_curseg_space(sbi, curseg)) {
3399 if (from_gc)
3400 get_atssr_segment(sbi, type, se->type,
3401 AT_SSR, se->mtime);
3402 else
3403 sit_i->s_ops->allocate_segment(sbi, type, false);
3404 }
3405
3406
3407
3408
3409
3410 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3411 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
3412
3413 up_write(&sit_i->sentry_lock);
3414
3415 if (page && IS_NODESEG(type)) {
3416 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
3417
3418 f2fs_inode_chksum_set(sbi, page);
3419 }
3420
3421 if (fio) {
3422 struct f2fs_bio_info *io;
3423
3424 if (F2FS_IO_ALIGNED(sbi))
3425 fio->retry = false;
3426
3427 INIT_LIST_HEAD(&fio->list);
3428 fio->in_list = true;
3429 io = sbi->write_io[fio->type] + fio->temp;
3430 spin_lock(&io->io_lock);
3431 list_add_tail(&fio->list, &io->io_list);
3432 spin_unlock(&io->io_lock);
3433 }
3434
3435 mutex_unlock(&curseg->curseg_mutex);
3436
3437 up_read(&SM_I(sbi)->curseg_lock);
3438}
3439
3440static void update_device_state(struct f2fs_io_info *fio)
3441{
3442 struct f2fs_sb_info *sbi = fio->sbi;
3443 unsigned int devidx;
3444
3445 if (!f2fs_is_multi_device(sbi))
3446 return;
3447
3448 devidx = f2fs_target_device_index(sbi, fio->new_blkaddr);
3449
3450
3451 f2fs_set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO);
3452
3453
3454 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
3455 spin_lock(&sbi->dev_lock);
3456 f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
3457 spin_unlock(&sbi->dev_lock);
3458 }
3459}
3460
3461static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
3462{
3463 int type = __get_segment_type(fio);
3464 bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);
3465
3466 if (keep_order)
3467 down_read(&fio->sbi->io_order_lock);
3468reallocate:
3469 f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
3470 &fio->new_blkaddr, sum, type, fio);
3471 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO)
3472 invalidate_mapping_pages(META_MAPPING(fio->sbi),
3473 fio->old_blkaddr, fio->old_blkaddr);
3474
3475
3476 f2fs_submit_page_write(fio);
3477 if (fio->retry) {
3478 fio->old_blkaddr = fio->new_blkaddr;
3479 goto reallocate;
3480 }
3481
3482 update_device_state(fio);
3483
3484 if (keep_order)
3485 up_read(&fio->sbi->io_order_lock);
3486}
3487
3488void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3489 enum iostat_type io_type)
3490{
3491 struct f2fs_io_info fio = {
3492 .sbi = sbi,
3493 .type = META,
3494 .temp = HOT,
3495 .op = REQ_OP_WRITE,
3496 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
3497 .old_blkaddr = page->index,
3498 .new_blkaddr = page->index,
3499 .page = page,
3500 .encrypted_page = NULL,
3501 .in_list = false,
3502 };
3503
3504 if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
3505 fio.op_flags &= ~REQ_META;
3506
3507 set_page_writeback(page);
3508 ClearPageError(page);
3509 f2fs_submit_page_write(&fio);
3510
3511 stat_inc_meta_count(sbi, page->index);
3512 f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);
3513}
3514
3515void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
3516{
3517 struct f2fs_summary sum;
3518
3519 set_summary(&sum, nid, 0, 0);
3520 do_write_page(&sum, fio);
3521
3522 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3523}
3524
3525void f2fs_outplace_write_data(struct dnode_of_data *dn,
3526 struct f2fs_io_info *fio)
3527{
3528 struct f2fs_sb_info *sbi = fio->sbi;
3529 struct f2fs_summary sum;
3530
3531 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
3532 set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
3533 do_write_page(&sum, fio);
3534 f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
3535
3536 f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);
3537}
3538
3539int f2fs_inplace_write_data(struct f2fs_io_info *fio)
3540{
3541 int err;
3542 struct f2fs_sb_info *sbi = fio->sbi;
3543 unsigned int segno;
3544
3545 fio->new_blkaddr = fio->old_blkaddr;
3546
3547 __get_segment_type(fio);
3548
3549 segno = GET_SEGNO(sbi, fio->new_blkaddr);
3550
3551 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
3552 set_sbi_flag(sbi, SBI_NEED_FSCK);
3553 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
3554 __func__, segno);
3555 err = -EFSCORRUPTED;
3556 goto drop_bio;
3557 }
3558
3559 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) || f2fs_cp_error(sbi)) {
3560 err = -EIO;
3561 goto drop_bio;
3562 }
3563
3564 stat_inc_inplace_blocks(fio->sbi);
3565
3566 if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE)))
3567 err = f2fs_merge_page_bio(fio);
3568 else
3569 err = f2fs_submit_page_bio(fio);
3570 if (!err) {
3571 update_device_state(fio);
3572 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3573 }
3574
3575 return err;
3576drop_bio:
3577 if (fio->bio && *(fio->bio)) {
3578 struct bio *bio = *(fio->bio);
3579
3580 bio->bi_status = BLK_STS_IOERR;
3581 bio_endio(bio);
3582 *(fio->bio) = NULL;
3583 }
3584 return err;
3585}
3586
3587static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
3588 unsigned int segno)
3589{
3590 int i;
3591
3592 for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
3593 if (CURSEG_I(sbi, i)->segno == segno)
3594 break;
3595 }
3596 return i;
3597}
3598
3599void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3600 block_t old_blkaddr, block_t new_blkaddr,
3601 bool recover_curseg, bool recover_newaddr,
3602 bool from_gc)
3603{
3604 struct sit_info *sit_i = SIT_I(sbi);
3605 struct curseg_info *curseg;
3606 unsigned int segno, old_cursegno;
3607 struct seg_entry *se;
3608 int type;
3609 unsigned short old_blkoff;
3610 unsigned char old_alloc_type;
3611
3612 segno = GET_SEGNO(sbi, new_blkaddr);
3613 se = get_seg_entry(sbi, segno);
3614 type = se->type;
3615
3616 down_write(&SM_I(sbi)->curseg_lock);
3617
3618 if (!recover_curseg) {
3619
3620 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
3621 if (old_blkaddr == NULL_ADDR)
3622 type = CURSEG_COLD_DATA;
3623 else
3624 type = CURSEG_WARM_DATA;
3625 }
3626 } else {
3627 if (IS_CURSEG(sbi, segno)) {
3628
3629 type = __f2fs_get_curseg(sbi, segno);
3630 f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
3631 } else {
3632 type = CURSEG_WARM_DATA;
3633 }
3634 }
3635
3636 f2fs_bug_on(sbi, !IS_DATASEG(type));
3637 curseg = CURSEG_I(sbi, type);
3638
3639 mutex_lock(&curseg->curseg_mutex);
3640 down_write(&sit_i->sentry_lock);
3641
3642 old_cursegno = curseg->segno;
3643 old_blkoff = curseg->next_blkoff;
3644 old_alloc_type = curseg->alloc_type;
3645
3646
3647 if (segno != curseg->segno) {
3648 curseg->next_segno = segno;
3649 change_curseg(sbi, type, true);
3650 }
3651
3652 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
3653 __add_sum_entry(sbi, type, sum);
3654
3655 if (!recover_curseg || recover_newaddr) {
3656 if (!from_gc)
3657 update_segment_mtime(sbi, new_blkaddr, 0);
3658 update_sit_entry(sbi, new_blkaddr, 1);
3659 }
3660 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
3661 invalidate_mapping_pages(META_MAPPING(sbi),
3662 old_blkaddr, old_blkaddr);
3663 if (!from_gc)
3664 update_segment_mtime(sbi, old_blkaddr, 0);
3665 update_sit_entry(sbi, old_blkaddr, -1);
3666 }
3667
3668 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3669 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
3670
3671 locate_dirty_segment(sbi, old_cursegno);
3672
3673 if (recover_curseg) {
3674 if (old_cursegno != curseg->segno) {
3675 curseg->next_segno = old_cursegno;
3676 change_curseg(sbi, type, true);
3677 }
3678 curseg->next_blkoff = old_blkoff;
3679 curseg->alloc_type = old_alloc_type;
3680 }
3681
3682 up_write(&sit_i->sentry_lock);
3683 mutex_unlock(&curseg->curseg_mutex);
3684 up_write(&SM_I(sbi)->curseg_lock);
3685}
3686
3687void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3688 block_t old_addr, block_t new_addr,
3689 unsigned char version, bool recover_curseg,
3690 bool recover_newaddr)
3691{
3692 struct f2fs_summary sum;
3693
3694 set_summary(&sum, dn->nid, dn->ofs_in_node, version);
3695
3696 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
3697 recover_curseg, recover_newaddr, false);
3698
3699 f2fs_update_data_blkaddr(dn, new_addr);
3700}
3701
3702void f2fs_wait_on_page_writeback(struct page *page,
3703 enum page_type type, bool ordered, bool locked)
3704{
3705 if (PageWriteback(page)) {
3706 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
3707
3708
3709 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
3710
3711 f2fs_submit_merged_ipu_write(sbi, NULL, page);
3712 if (ordered) {
3713 wait_on_page_writeback(page);
3714 f2fs_bug_on(sbi, locked && PageWriteback(page));
3715 } else {
3716 wait_for_stable_page(page);
3717 }
3718 }
3719}
3720
3721void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
3722{
3723 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3724 struct page *cpage;
3725
3726 if (!f2fs_post_read_required(inode))
3727 return;
3728
3729 if (!__is_valid_data_blkaddr(blkaddr))
3730 return;
3731
3732 cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
3733 if (cpage) {
3734 f2fs_wait_on_page_writeback(cpage, DATA, true, true);
3735 f2fs_put_page(cpage, 1);
3736 }
3737}
3738
3739void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3740 block_t len)
3741{
3742 block_t i;
3743
3744 for (i = 0; i < len; i++)
3745 f2fs_wait_on_block_writeback(inode, blkaddr + i);
3746}
3747
3748static int read_compacted_summaries(struct f2fs_sb_info *sbi)
3749{
3750 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3751 struct curseg_info *seg_i;
3752 unsigned char *kaddr;
3753 struct page *page;
3754 block_t start;
3755 int i, j, offset;
3756
3757 start = start_sum_block(sbi);
3758
3759 page = f2fs_get_meta_page(sbi, start++);
3760 if (IS_ERR(page))
3761 return PTR_ERR(page);
3762 kaddr = (unsigned char *)page_address(page);
3763
3764
3765 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3766 memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
3767
3768
3769 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3770 memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
3771 offset = 2 * SUM_JOURNAL_SIZE;
3772
3773
3774 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3775 unsigned short blk_off;
3776 unsigned int segno;
3777
3778 seg_i = CURSEG_I(sbi, i);
3779 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
3780 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
3781 seg_i->next_segno = segno;
3782 reset_curseg(sbi, i, 0);
3783 seg_i->alloc_type = ckpt->alloc_type[i];
3784 seg_i->next_blkoff = blk_off;
3785
3786 if (seg_i->alloc_type == SSR)
3787 blk_off = sbi->blocks_per_seg;
3788
3789 for (j = 0; j < blk_off; j++) {
3790 struct f2fs_summary *s;
3791
3792 s = (struct f2fs_summary *)(kaddr + offset);
3793 seg_i->sum_blk->entries[j] = *s;
3794 offset += SUMMARY_SIZE;
3795 if (offset + SUMMARY_SIZE <= PAGE_SIZE -
3796 SUM_FOOTER_SIZE)
3797 continue;
3798
3799 f2fs_put_page(page, 1);
3800 page = NULL;
3801
3802 page = f2fs_get_meta_page(sbi, start++);
3803 if (IS_ERR(page))
3804 return PTR_ERR(page);
3805 kaddr = (unsigned char *)page_address(page);
3806 offset = 0;
3807 }
3808 }
3809 f2fs_put_page(page, 1);
3810 return 0;
3811}
3812
3813static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
3814{
3815 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3816 struct f2fs_summary_block *sum;
3817 struct curseg_info *curseg;
3818 struct page *new;
3819 unsigned short blk_off;
3820 unsigned int segno = 0;
3821 block_t blk_addr = 0;
3822 int err = 0;
3823
3824
3825 if (IS_DATASEG(type)) {
3826 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
3827 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
3828 CURSEG_HOT_DATA]);
3829 if (__exist_node_summaries(sbi))
3830 blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
3831 else
3832 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
3833 } else {
3834 segno = le32_to_cpu(ckpt->cur_node_segno[type -
3835 CURSEG_HOT_NODE]);
3836 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
3837 CURSEG_HOT_NODE]);
3838 if (__exist_node_summaries(sbi))
3839 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
3840 type - CURSEG_HOT_NODE);
3841 else
3842 blk_addr = GET_SUM_BLOCK(sbi, segno);
3843 }
3844
3845 new = f2fs_get_meta_page(sbi, blk_addr);
3846 if (IS_ERR(new))
3847 return PTR_ERR(new);
3848 sum = (struct f2fs_summary_block *)page_address(new);
3849
3850 if (IS_NODESEG(type)) {
3851 if (__exist_node_summaries(sbi)) {
3852 struct f2fs_summary *ns = &sum->entries[0];
3853 int i;
3854
3855 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
3856 ns->version = 0;
3857 ns->ofs_in_node = 0;
3858 }
3859 } else {
3860 err = f2fs_restore_node_summary(sbi, segno, sum);
3861 if (err)
3862 goto out;
3863 }
3864 }
3865
3866
3867 curseg = CURSEG_I(sbi, type);
3868 mutex_lock(&curseg->curseg_mutex);
3869
3870
3871 down_write(&curseg->journal_rwsem);
3872 memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
3873 up_write(&curseg->journal_rwsem);
3874
3875 memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
3876 memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
3877 curseg->next_segno = segno;
3878 reset_curseg(sbi, type, 0);
3879 curseg->alloc_type = ckpt->alloc_type[type];
3880 curseg->next_blkoff = blk_off;
3881 mutex_unlock(&curseg->curseg_mutex);
3882out:
3883 f2fs_put_page(new, 1);
3884 return err;
3885}
3886
3887static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
3888{
3889 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
3890 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
3891 int type = CURSEG_HOT_DATA;
3892 int err;
3893
3894 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
3895 int npages = f2fs_npages_for_summary_flush(sbi, true);
3896
3897 if (npages >= 2)
3898 f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
3899 META_CP, true);
3900
3901
3902 err = read_compacted_summaries(sbi);
3903 if (err)
3904 return err;
3905 type = CURSEG_HOT_NODE;
3906 }
3907
3908 if (__exist_node_summaries(sbi))
3909 f2fs_ra_meta_pages(sbi,
3910 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
3911 NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
3912
3913 for (; type <= CURSEG_COLD_NODE; type++) {
3914 err = read_normal_summaries(sbi, type);
3915 if (err)
3916 return err;
3917 }
3918
3919
3920 if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
3921 sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
3922 f2fs_err(sbi, "invalid journal entries nats %u sits %u\n",
3923 nats_in_cursum(nat_j), sits_in_cursum(sit_j));
3924 return -EINVAL;
3925 }
3926
3927 return 0;
3928}
3929
3930static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
3931{
3932 struct page *page;
3933 unsigned char *kaddr;
3934 struct f2fs_summary *summary;
3935 struct curseg_info *seg_i;
3936 int written_size = 0;
3937 int i, j;
3938
3939 page = f2fs_grab_meta_page(sbi, blkaddr++);
3940 kaddr = (unsigned char *)page_address(page);
3941 memset(kaddr, 0, PAGE_SIZE);
3942
3943
3944 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3945 memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
3946 written_size += SUM_JOURNAL_SIZE;
3947
3948
3949 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3950 memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
3951 written_size += SUM_JOURNAL_SIZE;
3952
3953
3954 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3955 unsigned short blkoff;
3956
3957 seg_i = CURSEG_I(sbi, i);
3958 if (sbi->ckpt->alloc_type[i] == SSR)
3959 blkoff = sbi->blocks_per_seg;
3960 else
3961 blkoff = curseg_blkoff(sbi, i);
3962
3963 for (j = 0; j < blkoff; j++) {
3964 if (!page) {
3965 page = f2fs_grab_meta_page(sbi, blkaddr++);
3966 kaddr = (unsigned char *)page_address(page);
3967 memset(kaddr, 0, PAGE_SIZE);
3968 written_size = 0;
3969 }
3970 summary = (struct f2fs_summary *)(kaddr + written_size);
3971 *summary = seg_i->sum_blk->entries[j];
3972 written_size += SUMMARY_SIZE;
3973
3974 if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
3975 SUM_FOOTER_SIZE)
3976 continue;
3977
3978 set_page_dirty(page);
3979 f2fs_put_page(page, 1);
3980 page = NULL;
3981 }
3982 }
3983 if (page) {
3984 set_page_dirty(page);
3985 f2fs_put_page(page, 1);
3986 }
3987}
3988
3989static void write_normal_summaries(struct f2fs_sb_info *sbi,
3990 block_t blkaddr, int type)
3991{
3992 int i, end;
3993
3994 if (IS_DATASEG(type))
3995 end = type + NR_CURSEG_DATA_TYPE;
3996 else
3997 end = type + NR_CURSEG_NODE_TYPE;
3998
3999 for (i = type; i < end; i++)
4000 write_current_sum_page(sbi, i, blkaddr + (i - type));
4001}
4002
4003void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4004{
4005 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
4006 write_compacted_summaries(sbi, start_blk);
4007 else
4008 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
4009}
4010
4011void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4012{
4013 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
4014}
4015
4016int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
4017 unsigned int val, int alloc)
4018{
4019 int i;
4020
4021 if (type == NAT_JOURNAL) {
4022 for (i = 0; i < nats_in_cursum(journal); i++) {
4023 if (le32_to_cpu(nid_in_journal(journal, i)) == val)
4024 return i;
4025 }
4026 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
4027 return update_nats_in_cursum(journal, 1);
4028 } else if (type == SIT_JOURNAL) {
4029 for (i = 0; i < sits_in_cursum(journal); i++)
4030 if (le32_to_cpu(segno_in_journal(journal, i)) == val)
4031 return i;
4032 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
4033 return update_sits_in_cursum(journal, 1);
4034 }
4035 return -1;
4036}
4037
4038static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
4039 unsigned int segno)
4040{
4041 return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));
4042}
4043
4044static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
4045 unsigned int start)
4046{
4047 struct sit_info *sit_i = SIT_I(sbi);
4048 struct page *page;
4049 pgoff_t src_off, dst_off;
4050
4051 src_off = current_sit_addr(sbi, start);
4052 dst_off = next_sit_addr(sbi, src_off);
4053
4054 page = f2fs_grab_meta_page(sbi, dst_off);
4055 seg_info_to_sit_page(sbi, page, start);
4056
4057 set_page_dirty(page);
4058 set_to_next_sit(sit_i, start);
4059
4060 return page;
4061}
4062
4063static struct sit_entry_set *grab_sit_entry_set(void)
4064{
4065 struct sit_entry_set *ses =
4066 f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
4067
4068 ses->entry_cnt = 0;
4069 INIT_LIST_HEAD(&ses->set_list);
4070 return ses;
4071}
4072
4073static void release_sit_entry_set(struct sit_entry_set *ses)
4074{
4075 list_del(&ses->set_list);
4076 kmem_cache_free(sit_entry_set_slab, ses);
4077}
4078
4079static void adjust_sit_entry_set(struct sit_entry_set *ses,
4080 struct list_head *head)
4081{
4082 struct sit_entry_set *next = ses;
4083
4084 if (list_is_last(&ses->set_list, head))
4085 return;
4086
4087 list_for_each_entry_continue(next, head, set_list)
4088 if (ses->entry_cnt <= next->entry_cnt)
4089 break;
4090
4091 list_move_tail(&ses->set_list, &next->set_list);
4092}
4093
4094static void add_sit_entry(unsigned int segno, struct list_head *head)
4095{
4096 struct sit_entry_set *ses;
4097 unsigned int start_segno = START_SEGNO(segno);
4098
4099 list_for_each_entry(ses, head, set_list) {
4100 if (ses->start_segno == start_segno) {
4101 ses->entry_cnt++;
4102 adjust_sit_entry_set(ses, head);
4103 return;
4104 }
4105 }
4106
4107 ses = grab_sit_entry_set();
4108
4109 ses->start_segno = start_segno;
4110 ses->entry_cnt++;
4111 list_add(&ses->set_list, head);
4112}
4113
4114static void add_sits_in_set(struct f2fs_sb_info *sbi)
4115{
4116 struct f2fs_sm_info *sm_info = SM_I(sbi);
4117 struct list_head *set_list = &sm_info->sit_entry_set;
4118 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
4119 unsigned int segno;
4120
4121 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
4122 add_sit_entry(segno, set_list);
4123}
4124
4125static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
4126{
4127 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4128 struct f2fs_journal *journal = curseg->journal;
4129 int i;
4130
4131 down_write(&curseg->journal_rwsem);
4132 for (i = 0; i < sits_in_cursum(journal); i++) {
4133 unsigned int segno;
4134 bool dirtied;
4135
4136 segno = le32_to_cpu(segno_in_journal(journal, i));
4137 dirtied = __mark_sit_entry_dirty(sbi, segno);
4138
4139 if (!dirtied)
4140 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
4141 }
4142 update_sits_in_cursum(journal, -i);
4143 up_write(&curseg->journal_rwsem);
4144}
4145
4146
4147
4148
4149
4150void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
4151{
4152 struct sit_info *sit_i = SIT_I(sbi);
4153 unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
4154 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4155 struct f2fs_journal *journal = curseg->journal;
4156 struct sit_entry_set *ses, *tmp;
4157 struct list_head *head = &SM_I(sbi)->sit_entry_set;
4158 bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
4159 struct seg_entry *se;
4160
4161 down_write(&sit_i->sentry_lock);
4162
4163 if (!sit_i->dirty_sentries)
4164 goto out;
4165
4166
4167
4168
4169
4170 add_sits_in_set(sbi);
4171
4172
4173
4174
4175
4176
4177 if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) ||
4178 !to_journal)
4179 remove_sits_in_journal(sbi);
4180
4181
4182
4183
4184
4185
4186 list_for_each_entry_safe(ses, tmp, head, set_list) {
4187 struct page *page = NULL;
4188 struct f2fs_sit_block *raw_sit = NULL;
4189 unsigned int start_segno = ses->start_segno;
4190 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
4191 (unsigned long)MAIN_SEGS(sbi));
4192 unsigned int segno = start_segno;
4193
4194 if (to_journal &&
4195 !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
4196 to_journal = false;
4197
4198 if (to_journal) {
4199 down_write(&curseg->journal_rwsem);
4200 } else {
4201 page = get_next_sit_page(sbi, start_segno);
4202 raw_sit = page_address(page);
4203 }
4204
4205
4206 for_each_set_bit_from(segno, bitmap, end) {
4207 int offset, sit_offset;
4208
4209 se = get_seg_entry(sbi, segno);
4210#ifdef CONFIG_F2FS_CHECK_FS
4211 if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
4212 SIT_VBLOCK_MAP_SIZE))
4213 f2fs_bug_on(sbi, 1);
4214#endif
4215
4216
4217 if (!(cpc->reason & CP_DISCARD)) {
4218 cpc->trim_start = segno;
4219 add_discard_addrs(sbi, cpc, false);
4220 }
4221
4222 if (to_journal) {
4223 offset = f2fs_lookup_journal_in_cursum(journal,
4224 SIT_JOURNAL, segno, 1);
4225 f2fs_bug_on(sbi, offset < 0);
4226 segno_in_journal(journal, offset) =
4227 cpu_to_le32(segno);
4228 seg_info_to_raw_sit(se,
4229 &sit_in_journal(journal, offset));
4230 check_block_count(sbi, segno,
4231 &sit_in_journal(journal, offset));
4232 } else {
4233 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
4234 seg_info_to_raw_sit(se,
4235 &raw_sit->entries[sit_offset]);
4236 check_block_count(sbi, segno,
4237 &raw_sit->entries[sit_offset]);
4238 }
4239
4240 __clear_bit(segno, bitmap);
4241 sit_i->dirty_sentries--;
4242 ses->entry_cnt--;
4243 }
4244
4245 if (to_journal)
4246 up_write(&curseg->journal_rwsem);
4247 else
4248 f2fs_put_page(page, 1);
4249
4250 f2fs_bug_on(sbi, ses->entry_cnt);
4251 release_sit_entry_set(ses);
4252 }
4253
4254 f2fs_bug_on(sbi, !list_empty(head));
4255 f2fs_bug_on(sbi, sit_i->dirty_sentries);
4256out:
4257 if (cpc->reason & CP_DISCARD) {
4258 __u64 trim_start = cpc->trim_start;
4259
4260 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
4261 add_discard_addrs(sbi, cpc, false);
4262
4263 cpc->trim_start = trim_start;
4264 }
4265 up_write(&sit_i->sentry_lock);
4266
4267 set_prefree_as_free_segments(sbi);
4268}
4269
4270static int build_sit_info(struct f2fs_sb_info *sbi)
4271{
4272 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4273 struct sit_info *sit_i;
4274 unsigned int sit_segs, start;
4275 char *src_bitmap, *bitmap;
4276 unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
4277
4278
4279 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
4280 if (!sit_i)
4281 return -ENOMEM;
4282
4283 SM_I(sbi)->sit_info = sit_i;
4284
4285 sit_i->sentries =
4286 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
4287 MAIN_SEGS(sbi)),
4288 GFP_KERNEL);
4289 if (!sit_i->sentries)
4290 return -ENOMEM;
4291
4292 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4293 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
4294 GFP_KERNEL);
4295 if (!sit_i->dirty_sentries_bitmap)
4296 return -ENOMEM;
4297
4298#ifdef CONFIG_F2FS_CHECK_FS
4299 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 4;
4300#else
4301 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 3;
4302#endif
4303 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4304 if (!sit_i->bitmap)
4305 return -ENOMEM;
4306
4307 bitmap = sit_i->bitmap;
4308
4309 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4310 sit_i->sentries[start].cur_valid_map = bitmap;
4311 bitmap += SIT_VBLOCK_MAP_SIZE;
4312
4313 sit_i->sentries[start].ckpt_valid_map = bitmap;
4314 bitmap += SIT_VBLOCK_MAP_SIZE;
4315
4316#ifdef CONFIG_F2FS_CHECK_FS
4317 sit_i->sentries[start].cur_valid_map_mir = bitmap;
4318 bitmap += SIT_VBLOCK_MAP_SIZE;
4319#endif
4320
4321 sit_i->sentries[start].discard_map = bitmap;
4322 bitmap += SIT_VBLOCK_MAP_SIZE;
4323 }
4324
4325 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4326 if (!sit_i->tmp_map)
4327 return -ENOMEM;
4328
4329 if (__is_large_section(sbi)) {
4330 sit_i->sec_entries =
4331 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
4332 MAIN_SECS(sbi)),
4333 GFP_KERNEL);
4334 if (!sit_i->sec_entries)
4335 return -ENOMEM;
4336 }
4337
4338
4339 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
4340
4341
4342 sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
4343 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
4344
4345 sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
4346 if (!sit_i->sit_bitmap)
4347 return -ENOMEM;
4348
4349#ifdef CONFIG_F2FS_CHECK_FS
4350 sit_i->sit_bitmap_mir = kmemdup(src_bitmap,
4351 sit_bitmap_size, GFP_KERNEL);
4352 if (!sit_i->sit_bitmap_mir)
4353 return -ENOMEM;
4354
4355 sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
4356 main_bitmap_size, GFP_KERNEL);
4357 if (!sit_i->invalid_segmap)
4358 return -ENOMEM;
4359#endif
4360
4361
4362 sit_i->s_ops = &default_salloc_ops;
4363
4364 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
4365 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
4366 sit_i->written_valid_blocks = 0;
4367 sit_i->bitmap_size = sit_bitmap_size;
4368 sit_i->dirty_sentries = 0;
4369 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
4370 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
4371 sit_i->mounted_time = ktime_get_boottime_seconds();
4372 init_rwsem(&sit_i->sentry_lock);
4373 return 0;
4374}
4375
4376static int build_free_segmap(struct f2fs_sb_info *sbi)
4377{
4378 struct free_segmap_info *free_i;
4379 unsigned int bitmap_size, sec_bitmap_size;
4380
4381
4382 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
4383 if (!free_i)
4384 return -ENOMEM;
4385
4386 SM_I(sbi)->free_info = free_i;
4387
4388 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4389 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
4390 if (!free_i->free_segmap)
4391 return -ENOMEM;
4392
4393 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4394 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
4395 if (!free_i->free_secmap)
4396 return -ENOMEM;
4397
4398
4399 memset(free_i->free_segmap, 0xff, bitmap_size);
4400 memset(free_i->free_secmap, 0xff, sec_bitmap_size);
4401
4402
4403 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
4404 free_i->free_segments = 0;
4405 free_i->free_sections = 0;
4406 spin_lock_init(&free_i->segmap_lock);
4407 return 0;
4408}
4409
4410static int build_curseg(struct f2fs_sb_info *sbi)
4411{
4412 struct curseg_info *array;
4413 int i;
4414
4415 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
4416 sizeof(*array)), GFP_KERNEL);
4417 if (!array)
4418 return -ENOMEM;
4419
4420 SM_I(sbi)->curseg_array = array;
4421
4422 for (i = 0; i < NO_CHECK_TYPE; i++) {
4423 mutex_init(&array[i].curseg_mutex);
4424 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
4425 if (!array[i].sum_blk)
4426 return -ENOMEM;
4427 init_rwsem(&array[i].journal_rwsem);
4428 array[i].journal = f2fs_kzalloc(sbi,
4429 sizeof(struct f2fs_journal), GFP_KERNEL);
4430 if (!array[i].journal)
4431 return -ENOMEM;
4432 if (i < NR_PERSISTENT_LOG)
4433 array[i].seg_type = CURSEG_HOT_DATA + i;
4434 else if (i == CURSEG_COLD_DATA_PINNED)
4435 array[i].seg_type = CURSEG_COLD_DATA;
4436 else if (i == CURSEG_ALL_DATA_ATGC)
4437 array[i].seg_type = CURSEG_COLD_DATA;
4438 array[i].segno = NULL_SEGNO;
4439 array[i].next_blkoff = 0;
4440 array[i].inited = false;
4441 }
4442 return restore_curseg_summaries(sbi);
4443}
4444
4445static int build_sit_entries(struct f2fs_sb_info *sbi)
4446{
4447 struct sit_info *sit_i = SIT_I(sbi);
4448 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4449 struct f2fs_journal *journal = curseg->journal;
4450 struct seg_entry *se;
4451 struct f2fs_sit_entry sit;
4452 int sit_blk_cnt = SIT_BLK_CNT(sbi);
4453 unsigned int i, start, end;
4454 unsigned int readed, start_blk = 0;
4455 int err = 0;
4456 block_t total_node_blocks = 0;
4457
4458 do {
4459 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS,
4460 META_SIT, true);
4461
4462 start = start_blk * sit_i->sents_per_block;
4463 end = (start_blk + readed) * sit_i->sents_per_block;
4464
4465 for (; start < end && start < MAIN_SEGS(sbi); start++) {
4466 struct f2fs_sit_block *sit_blk;
4467 struct page *page;
4468
4469 se = &sit_i->sentries[start];
4470 page = get_current_sit_page(sbi, start);
4471 if (IS_ERR(page))
4472 return PTR_ERR(page);
4473 sit_blk = (struct f2fs_sit_block *)page_address(page);
4474 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
4475 f2fs_put_page(page, 1);
4476
4477 err = check_block_count(sbi, start, &sit);
4478 if (err)
4479 return err;
4480 seg_info_from_raw_sit(se, &sit);
4481 if (IS_NODESEG(se->type))
4482 total_node_blocks += se->valid_blocks;
4483
4484
4485 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4486 memset(se->discard_map, 0xff,
4487 SIT_VBLOCK_MAP_SIZE);
4488 } else {
4489 memcpy(se->discard_map,
4490 se->cur_valid_map,
4491 SIT_VBLOCK_MAP_SIZE);
4492 sbi->discard_blks +=
4493 sbi->blocks_per_seg -
4494 se->valid_blocks;
4495 }
4496
4497 if (__is_large_section(sbi))
4498 get_sec_entry(sbi, start)->valid_blocks +=
4499 se->valid_blocks;
4500 }
4501 start_blk += readed;
4502 } while (start_blk < sit_blk_cnt);
4503
4504 down_read(&curseg->journal_rwsem);
4505 for (i = 0; i < sits_in_cursum(journal); i++) {
4506 unsigned int old_valid_blocks;
4507
4508 start = le32_to_cpu(segno_in_journal(journal, i));
4509 if (start >= MAIN_SEGS(sbi)) {
4510 f2fs_err(sbi, "Wrong journal entry on segno %u",
4511 start);
4512 err = -EFSCORRUPTED;
4513 break;
4514 }
4515
4516 se = &sit_i->sentries[start];
4517 sit = sit_in_journal(journal, i);
4518
4519 old_valid_blocks = se->valid_blocks;
4520 if (IS_NODESEG(se->type))
4521 total_node_blocks -= old_valid_blocks;
4522
4523 err = check_block_count(sbi, start, &sit);
4524 if (err)
4525 break;
4526 seg_info_from_raw_sit(se, &sit);
4527 if (IS_NODESEG(se->type))
4528 total_node_blocks += se->valid_blocks;
4529
4530 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4531 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
4532 } else {
4533 memcpy(se->discard_map, se->cur_valid_map,
4534 SIT_VBLOCK_MAP_SIZE);
4535 sbi->discard_blks += old_valid_blocks;
4536 sbi->discard_blks -= se->valid_blocks;
4537 }
4538
4539 if (__is_large_section(sbi)) {
4540 get_sec_entry(sbi, start)->valid_blocks +=
4541 se->valid_blocks;
4542 get_sec_entry(sbi, start)->valid_blocks -=
4543 old_valid_blocks;
4544 }
4545 }
4546 up_read(&curseg->journal_rwsem);
4547
4548 if (!err && total_node_blocks != valid_node_count(sbi)) {
4549 f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
4550 total_node_blocks, valid_node_count(sbi));
4551 err = -EFSCORRUPTED;
4552 }
4553
4554 return err;
4555}
4556
4557static void init_free_segmap(struct f2fs_sb_info *sbi)
4558{
4559 unsigned int start;
4560 int type;
4561 struct seg_entry *sentry;
4562
4563 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4564 if (f2fs_usable_blks_in_seg(sbi, start) == 0)
4565 continue;
4566 sentry = get_seg_entry(sbi, start);
4567 if (!sentry->valid_blocks)
4568 __set_free(sbi, start);
4569 else
4570 SIT_I(sbi)->written_valid_blocks +=
4571 sentry->valid_blocks;
4572 }
4573
4574
4575 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
4576 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
4577
4578 __set_test_and_inuse(sbi, curseg_t->segno);
4579 }
4580}
4581
4582static void init_dirty_segmap(struct f2fs_sb_info *sbi)
4583{
4584 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4585 struct free_segmap_info *free_i = FREE_I(sbi);
4586 unsigned int segno = 0, offset = 0, secno;
4587 block_t valid_blocks, usable_blks_in_seg;
4588 block_t blks_per_sec = BLKS_PER_SEC(sbi);
4589
4590 while (1) {
4591
4592 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
4593 if (segno >= MAIN_SEGS(sbi))
4594 break;
4595 offset = segno + 1;
4596 valid_blocks = get_valid_blocks(sbi, segno, false);
4597 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
4598 if (valid_blocks == usable_blks_in_seg || !valid_blocks)
4599 continue;
4600 if (valid_blocks > usable_blks_in_seg) {
4601 f2fs_bug_on(sbi, 1);
4602 continue;
4603 }
4604 mutex_lock(&dirty_i->seglist_lock);
4605 __locate_dirty_segment(sbi, segno, DIRTY);
4606 mutex_unlock(&dirty_i->seglist_lock);
4607 }
4608
4609 if (!__is_large_section(sbi))
4610 return;
4611
4612 mutex_lock(&dirty_i->seglist_lock);
4613 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
4614 valid_blocks = get_valid_blocks(sbi, segno, true);
4615 secno = GET_SEC_FROM_SEG(sbi, segno);
4616
4617 if (!valid_blocks || valid_blocks == blks_per_sec)
4618 continue;
4619 if (IS_CURSEC(sbi, secno))
4620 continue;
4621 set_bit(secno, dirty_i->dirty_secmap);
4622 }
4623 mutex_unlock(&dirty_i->seglist_lock);
4624}
4625
4626static int init_victim_secmap(struct f2fs_sb_info *sbi)
4627{
4628 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4629 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4630
4631 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4632 if (!dirty_i->victim_secmap)
4633 return -ENOMEM;
4634 return 0;
4635}
4636
4637static int build_dirty_segmap(struct f2fs_sb_info *sbi)
4638{
4639 struct dirty_seglist_info *dirty_i;
4640 unsigned int bitmap_size, i;
4641
4642
4643 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
4644 GFP_KERNEL);
4645 if (!dirty_i)
4646 return -ENOMEM;
4647
4648 SM_I(sbi)->dirty_info = dirty_i;
4649 mutex_init(&dirty_i->seglist_lock);
4650
4651 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4652
4653 for (i = 0; i < NR_DIRTY_TYPE; i++) {
4654 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
4655 GFP_KERNEL);
4656 if (!dirty_i->dirty_segmap[i])
4657 return -ENOMEM;
4658 }
4659
4660 if (__is_large_section(sbi)) {
4661 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4662 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
4663 bitmap_size, GFP_KERNEL);
4664 if (!dirty_i->dirty_secmap)
4665 return -ENOMEM;
4666 }
4667
4668 init_dirty_segmap(sbi);
4669 return init_victim_secmap(sbi);
4670}
4671
4672static int sanity_check_curseg(struct f2fs_sb_info *sbi)
4673{
4674 int i;
4675
4676
4677
4678
4679
4680 for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4681 struct curseg_info *curseg = CURSEG_I(sbi, i);
4682 struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
4683 unsigned int blkofs = curseg->next_blkoff;
4684
4685 sanity_check_seg_type(sbi, curseg->seg_type);
4686
4687 if (f2fs_test_bit(blkofs, se->cur_valid_map))
4688 goto out;
4689
4690 if (curseg->alloc_type == SSR)
4691 continue;
4692
4693 for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {
4694 if (!f2fs_test_bit(blkofs, se->cur_valid_map))
4695 continue;
4696out:
4697 f2fs_err(sbi,
4698 "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
4699 i, curseg->segno, curseg->alloc_type,
4700 curseg->next_blkoff, blkofs);
4701 return -EFSCORRUPTED;
4702 }
4703 }
4704 return 0;
4705}
4706
4707#ifdef CONFIG_BLK_DEV_ZONED
4708
4709static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
4710 struct f2fs_dev_info *fdev,
4711 struct blk_zone *zone)
4712{
4713 unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno;
4714 block_t zone_block, wp_block, last_valid_block;
4715 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4716 int i, s, b, ret;
4717 struct seg_entry *se;
4718
4719 if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4720 return 0;
4721
4722 wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block);
4723 wp_segno = GET_SEGNO(sbi, wp_block);
4724 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4725 zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
4726 zone_segno = GET_SEGNO(sbi, zone_block);
4727 zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);
4728
4729 if (zone_segno >= MAIN_SEGS(sbi))
4730 return 0;
4731
4732
4733
4734
4735
4736 for (i = 0; i < NO_CHECK_TYPE; i++)
4737 if (zone_secno == GET_SEC_FROM_SEG(sbi,
4738 CURSEG_I(sbi, i)->segno))
4739 return 0;
4740
4741
4742
4743
4744 last_valid_block = zone_block - 1;
4745 for (s = sbi->segs_per_sec - 1; s >= 0; s--) {
4746 segno = zone_segno + s;
4747 se = get_seg_entry(sbi, segno);
4748 for (b = sbi->blocks_per_seg - 1; b >= 0; b--)
4749 if (f2fs_test_bit(b, se->cur_valid_map)) {
4750 last_valid_block = START_BLOCK(sbi, segno) + b;
4751 break;
4752 }
4753 if (last_valid_block >= zone_block)
4754 break;
4755 }
4756
4757
4758
4759
4760
4761
4762
4763 if (last_valid_block >= wp_block) {
4764 f2fs_notice(sbi, "Valid block beyond write pointer: "
4765 "valid block[0x%x,0x%x] wp[0x%x,0x%x]",
4766 GET_SEGNO(sbi, last_valid_block),
4767 GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),
4768 wp_segno, wp_blkoff);
4769 return 0;
4770 }
4771
4772
4773
4774
4775
4776 if (last_valid_block + 1 == zone_block && zone->wp != zone->start) {
4777 f2fs_notice(sbi,
4778 "Zone without valid block has non-zero write "
4779 "pointer. Reset the write pointer: wp[0x%x,0x%x]",
4780 wp_segno, wp_blkoff);
4781 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
4782 zone->len >> log_sectors_per_block);
4783 if (ret) {
4784 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4785 fdev->path, ret);
4786 return ret;
4787 }
4788 }
4789
4790 return 0;
4791}
4792
4793static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
4794 block_t zone_blkaddr)
4795{
4796 int i;
4797
4798 for (i = 0; i < sbi->s_ndevs; i++) {
4799 if (!bdev_is_zoned(FDEV(i).bdev))
4800 continue;
4801 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&
4802 zone_blkaddr <= FDEV(i).end_blk))
4803 return &FDEV(i);
4804 }
4805
4806 return NULL;
4807}
4808
4809static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
4810 void *data)
4811{
4812 memcpy(data, zone, sizeof(struct blk_zone));
4813 return 0;
4814}
4815
4816static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
4817{
4818 struct curseg_info *cs = CURSEG_I(sbi, type);
4819 struct f2fs_dev_info *zbd;
4820 struct blk_zone zone;
4821 unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
4822 block_t cs_zone_block, wp_block;
4823 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4824 sector_t zone_sector;
4825 int err;
4826
4827 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4828 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4829
4830 zbd = get_target_zoned_dev(sbi, cs_zone_block);
4831 if (!zbd)
4832 return 0;
4833
4834
4835 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4836 << log_sectors_per_block;
4837 err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4838 report_one_zone_cb, &zone);
4839 if (err != 1) {
4840 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4841 zbd->path, err);
4842 return err;
4843 }
4844
4845 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4846 return 0;
4847
4848 wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
4849 wp_segno = GET_SEGNO(sbi, wp_block);
4850 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4851 wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
4852
4853 if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
4854 wp_sector_off == 0)
4855 return 0;
4856
4857 f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
4858 "curseg[0x%x,0x%x] wp[0x%x,0x%x]",
4859 type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff);
4860
4861 f2fs_notice(sbi, "Assign new section to curseg[%d]: "
4862 "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff);
4863
4864 f2fs_allocate_new_section(sbi, type, true);
4865
4866
4867 if (check_zone_write_pointer(sbi, zbd, &zone))
4868 return -EIO;
4869
4870
4871 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4872 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4873
4874 zbd = get_target_zoned_dev(sbi, cs_zone_block);
4875 if (!zbd)
4876 return 0;
4877
4878 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4879 << log_sectors_per_block;
4880 err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4881 report_one_zone_cb, &zone);
4882 if (err != 1) {
4883 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4884 zbd->path, err);
4885 return err;
4886 }
4887
4888 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4889 return 0;
4890
4891 if (zone.wp != zone.start) {
4892 f2fs_notice(sbi,
4893 "New zone for curseg[%d] is not yet discarded. "
4894 "Reset the zone: curseg[0x%x,0x%x]",
4895 type, cs->segno, cs->next_blkoff);
4896 err = __f2fs_issue_discard_zone(sbi, zbd->bdev,
4897 zone_sector >> log_sectors_per_block,
4898 zone.len >> log_sectors_per_block);
4899 if (err) {
4900 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4901 zbd->path, err);
4902 return err;
4903 }
4904 }
4905
4906 return 0;
4907}
4908
4909int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
4910{
4911 int i, ret;
4912
4913 for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4914 ret = fix_curseg_write_pointer(sbi, i);
4915 if (ret)
4916 return ret;
4917 }
4918
4919 return 0;
4920}
4921
4922struct check_zone_write_pointer_args {
4923 struct f2fs_sb_info *sbi;
4924 struct f2fs_dev_info *fdev;
4925};
4926
4927static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
4928 void *data)
4929{
4930 struct check_zone_write_pointer_args *args;
4931
4932 args = (struct check_zone_write_pointer_args *)data;
4933
4934 return check_zone_write_pointer(args->sbi, args->fdev, zone);
4935}
4936
4937int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
4938{
4939 int i, ret;
4940 struct check_zone_write_pointer_args args;
4941
4942 for (i = 0; i < sbi->s_ndevs; i++) {
4943 if (!bdev_is_zoned(FDEV(i).bdev))
4944 continue;
4945
4946 args.sbi = sbi;
4947 args.fdev = &FDEV(i);
4948 ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
4949 check_zone_write_pointer_cb, &args);
4950 if (ret < 0)
4951 return ret;
4952 }
4953
4954 return 0;
4955}
4956
4957static bool is_conv_zone(struct f2fs_sb_info *sbi, unsigned int zone_idx,
4958 unsigned int dev_idx)
4959{
4960 if (!bdev_is_zoned(FDEV(dev_idx).bdev))
4961 return true;
4962 return !test_bit(zone_idx, FDEV(dev_idx).blkz_seq);
4963}
4964
4965
4966static unsigned int get_zone_idx(struct f2fs_sb_info *sbi, unsigned int secno,
4967 int dev_idx)
4968{
4969 block_t sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
4970
4971 return (sec_start_blkaddr - FDEV(dev_idx).start_blk) >>
4972 sbi->log_blocks_per_blkz;
4973}
4974
4975
4976
4977
4978
4979static inline unsigned int f2fs_usable_zone_segs_in_sec(
4980 struct f2fs_sb_info *sbi, unsigned int segno)
4981{
4982 unsigned int dev_idx, zone_idx, unusable_segs_in_sec;
4983
4984 dev_idx = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno));
4985 zone_idx = get_zone_idx(sbi, GET_SEC_FROM_SEG(sbi, segno), dev_idx);
4986
4987
4988 if (is_conv_zone(sbi, zone_idx, dev_idx))
4989 return sbi->segs_per_sec;
4990
4991
4992
4993
4994
4995 if (!FDEV(dev_idx).zone_capacity_blocks)
4996 return sbi->segs_per_sec;
4997
4998
4999 unusable_segs_in_sec = (sbi->blocks_per_blkz -
5000 FDEV(dev_idx).zone_capacity_blocks[zone_idx]) >>
5001 sbi->log_blocks_per_seg;
5002 return sbi->segs_per_sec - unusable_segs_in_sec;
5003}
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013static inline unsigned int f2fs_usable_zone_blks_in_seg(
5014 struct f2fs_sb_info *sbi, unsigned int segno)
5015{
5016 block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
5017 unsigned int zone_idx, dev_idx, secno;
5018
5019 secno = GET_SEC_FROM_SEG(sbi, segno);
5020 seg_start = START_BLOCK(sbi, segno);
5021 dev_idx = f2fs_target_device_index(sbi, seg_start);
5022 zone_idx = get_zone_idx(sbi, secno, dev_idx);
5023
5024
5025
5026
5027
5028 if (is_conv_zone(sbi, zone_idx, dev_idx))
5029 return sbi->blocks_per_seg;
5030
5031 if (!FDEV(dev_idx).zone_capacity_blocks)
5032 return sbi->blocks_per_seg;
5033
5034 sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
5035 sec_cap_blkaddr = sec_start_blkaddr +
5036 FDEV(dev_idx).zone_capacity_blocks[zone_idx];
5037
5038
5039
5040
5041
5042
5043
5044 if (seg_start >= sec_cap_blkaddr)
5045 return 0;
5046 if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)
5047 return sec_cap_blkaddr - seg_start;
5048
5049 return sbi->blocks_per_seg;
5050}
5051#else
5052int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
5053{
5054 return 0;
5055}
5056
5057int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
5058{
5059 return 0;
5060}
5061
5062static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
5063 unsigned int segno)
5064{
5065 return 0;
5066}
5067
5068static inline unsigned int f2fs_usable_zone_segs_in_sec(struct f2fs_sb_info *sbi,
5069 unsigned int segno)
5070{
5071 return 0;
5072}
5073#endif
5074unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
5075 unsigned int segno)
5076{
5077 if (f2fs_sb_has_blkzoned(sbi))
5078 return f2fs_usable_zone_blks_in_seg(sbi, segno);
5079
5080 return sbi->blocks_per_seg;
5081}
5082
5083unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
5084 unsigned int segno)
5085{
5086 if (f2fs_sb_has_blkzoned(sbi))
5087 return f2fs_usable_zone_segs_in_sec(sbi, segno);
5088
5089 return sbi->segs_per_sec;
5090}
5091
5092
5093
5094
5095static void init_min_max_mtime(struct f2fs_sb_info *sbi)
5096{
5097 struct sit_info *sit_i = SIT_I(sbi);
5098 unsigned int segno;
5099
5100 down_write(&sit_i->sentry_lock);
5101
5102 sit_i->min_mtime = ULLONG_MAX;
5103
5104 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
5105 unsigned int i;
5106 unsigned long long mtime = 0;
5107
5108 for (i = 0; i < sbi->segs_per_sec; i++)
5109 mtime += get_seg_entry(sbi, segno + i)->mtime;
5110
5111 mtime = div_u64(mtime, sbi->segs_per_sec);
5112
5113 if (sit_i->min_mtime > mtime)
5114 sit_i->min_mtime = mtime;
5115 }
5116 sit_i->max_mtime = get_mtime(sbi, false);
5117 sit_i->dirty_max_mtime = 0;
5118 up_write(&sit_i->sentry_lock);
5119}
5120
5121int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
5122{
5123 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
5124 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
5125 struct f2fs_sm_info *sm_info;
5126 int err;
5127
5128 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
5129 if (!sm_info)
5130 return -ENOMEM;
5131
5132
5133 sbi->sm_info = sm_info;
5134 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
5135 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
5136 sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
5137 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
5138 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
5139 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
5140 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
5141 sm_info->rec_prefree_segments = sm_info->main_segments *
5142 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
5143 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
5144 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
5145
5146 if (!f2fs_lfs_mode(sbi))
5147 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
5148 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
5149 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
5150 sm_info->min_seq_blocks = sbi->blocks_per_seg * sbi->segs_per_sec;
5151 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
5152 sm_info->min_ssr_sections = reserved_sections(sbi);
5153
5154 INIT_LIST_HEAD(&sm_info->sit_entry_set);
5155
5156 init_rwsem(&sm_info->curseg_lock);
5157
5158 if (!f2fs_readonly(sbi->sb)) {
5159 err = f2fs_create_flush_cmd_control(sbi);
5160 if (err)
5161 return err;
5162 }
5163
5164 err = create_discard_cmd_control(sbi);
5165 if (err)
5166 return err;
5167
5168 err = build_sit_info(sbi);
5169 if (err)
5170 return err;
5171 err = build_free_segmap(sbi);
5172 if (err)
5173 return err;
5174 err = build_curseg(sbi);
5175 if (err)
5176 return err;
5177
5178
5179 err = build_sit_entries(sbi);
5180 if (err)
5181 return err;
5182
5183 init_free_segmap(sbi);
5184 err = build_dirty_segmap(sbi);
5185 if (err)
5186 return err;
5187
5188 err = sanity_check_curseg(sbi);
5189 if (err)
5190 return err;
5191
5192 init_min_max_mtime(sbi);
5193 return 0;
5194}
5195
5196static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
5197 enum dirty_type dirty_type)
5198{
5199 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5200
5201 mutex_lock(&dirty_i->seglist_lock);
5202 kvfree(dirty_i->dirty_segmap[dirty_type]);
5203 dirty_i->nr_dirty[dirty_type] = 0;
5204 mutex_unlock(&dirty_i->seglist_lock);
5205}
5206
5207static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
5208{
5209 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5210
5211 kvfree(dirty_i->victim_secmap);
5212}
5213
5214static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
5215{
5216 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5217 int i;
5218
5219 if (!dirty_i)
5220 return;
5221
5222
5223 for (i = 0; i < NR_DIRTY_TYPE; i++)
5224 discard_dirty_segmap(sbi, i);
5225
5226 if (__is_large_section(sbi)) {
5227 mutex_lock(&dirty_i->seglist_lock);
5228 kvfree(dirty_i->dirty_secmap);
5229 mutex_unlock(&dirty_i->seglist_lock);
5230 }
5231
5232 destroy_victim_secmap(sbi);
5233 SM_I(sbi)->dirty_info = NULL;
5234 kfree(dirty_i);
5235}
5236
5237static void destroy_curseg(struct f2fs_sb_info *sbi)
5238{
5239 struct curseg_info *array = SM_I(sbi)->curseg_array;
5240 int i;
5241
5242 if (!array)
5243 return;
5244 SM_I(sbi)->curseg_array = NULL;
5245 for (i = 0; i < NR_CURSEG_TYPE; i++) {
5246 kfree(array[i].sum_blk);
5247 kfree(array[i].journal);
5248 }
5249 kfree(array);
5250}
5251
5252static void destroy_free_segmap(struct f2fs_sb_info *sbi)
5253{
5254 struct free_segmap_info *free_i = SM_I(sbi)->free_info;
5255
5256 if (!free_i)
5257 return;
5258 SM_I(sbi)->free_info = NULL;
5259 kvfree(free_i->free_segmap);
5260 kvfree(free_i->free_secmap);
5261 kfree(free_i);
5262}
5263
5264static void destroy_sit_info(struct f2fs_sb_info *sbi)
5265{
5266 struct sit_info *sit_i = SIT_I(sbi);
5267
5268 if (!sit_i)
5269 return;
5270
5271 if (sit_i->sentries)
5272 kvfree(sit_i->bitmap);
5273 kfree(sit_i->tmp_map);
5274
5275 kvfree(sit_i->sentries);
5276 kvfree(sit_i->sec_entries);
5277 kvfree(sit_i->dirty_sentries_bitmap);
5278
5279 SM_I(sbi)->sit_info = NULL;
5280 kvfree(sit_i->sit_bitmap);
5281#ifdef CONFIG_F2FS_CHECK_FS
5282 kvfree(sit_i->sit_bitmap_mir);
5283 kvfree(sit_i->invalid_segmap);
5284#endif
5285 kfree(sit_i);
5286}
5287
5288void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
5289{
5290 struct f2fs_sm_info *sm_info = SM_I(sbi);
5291
5292 if (!sm_info)
5293 return;
5294 f2fs_destroy_flush_cmd_control(sbi, true);
5295 destroy_discard_cmd_control(sbi);
5296 destroy_dirty_segmap(sbi);
5297 destroy_curseg(sbi);
5298 destroy_free_segmap(sbi);
5299 destroy_sit_info(sbi);
5300 sbi->sm_info = NULL;
5301 kfree(sm_info);
5302}
5303
5304int __init f2fs_create_segment_manager_caches(void)
5305{
5306 discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry",
5307 sizeof(struct discard_entry));
5308 if (!discard_entry_slab)
5309 goto fail;
5310
5311 discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd",
5312 sizeof(struct discard_cmd));
5313 if (!discard_cmd_slab)
5314 goto destroy_discard_entry;
5315
5316 sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set",
5317 sizeof(struct sit_entry_set));
5318 if (!sit_entry_set_slab)
5319 goto destroy_discard_cmd;
5320
5321 inmem_entry_slab = f2fs_kmem_cache_create("f2fs_inmem_page_entry",
5322 sizeof(struct inmem_pages));
5323 if (!inmem_entry_slab)
5324 goto destroy_sit_entry_set;
5325 return 0;
5326
5327destroy_sit_entry_set:
5328 kmem_cache_destroy(sit_entry_set_slab);
5329destroy_discard_cmd:
5330 kmem_cache_destroy(discard_cmd_slab);
5331destroy_discard_entry:
5332 kmem_cache_destroy(discard_entry_slab);
5333fail:
5334 return -ENOMEM;
5335}
5336
5337void f2fs_destroy_segment_manager_caches(void)
5338{
5339 kmem_cache_destroy(sit_entry_set_slab);
5340 kmem_cache_destroy(discard_cmd_slab);
5341 kmem_cache_destroy(discard_entry_slab);
5342 kmem_cache_destroy(inmem_entry_slab);
5343}
5344
