1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
9 #include <linux/f2fs_fs.h>
10 #include <linux/bio.h>
11 #include <linux/blkdev.h>
12 #include <linux/sched/mm.h>
13 #include <linux/prefetch.h>
14 #include <linux/kthread.h>
15 #include <linux/swap.h>
16 #include <linux/timer.h>
17 #include <linux/freezer.h>
18 #include <linux/sched/signal.h>
19 #include <linux/random.h>
26 #include <trace/events/f2fs.h>
28 #define __reverse_ffz(x) __reverse_ffs(~(x))
30 static struct kmem_cache
*discard_entry_slab
;
31 static struct kmem_cache
*discard_cmd_slab
;
32 static struct kmem_cache
*sit_entry_set_slab
;
33 static struct kmem_cache
*revoke_entry_slab
;
35 static unsigned long __reverse_ulong(unsigned char *str
)
37 unsigned long tmp
= 0;
38 int shift
= 24, idx
= 0;
40 #if BITS_PER_LONG == 64
44 tmp
|= (unsigned long)str
[idx
++] << shift
;
45 shift
-= BITS_PER_BYTE
;
51 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
52 * MSB and LSB are reversed in a byte by f2fs_set_bit.
54 static inline unsigned long __reverse_ffs(unsigned long word
)
58 #if BITS_PER_LONG == 64
59 if ((word
& 0xffffffff00000000UL
) == 0)
64 if ((word
& 0xffff0000) == 0)
69 if ((word
& 0xff00) == 0)
74 if ((word
& 0xf0) == 0)
79 if ((word
& 0xc) == 0)
84 if ((word
& 0x2) == 0)
90 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
91 * f2fs_set_bit makes MSB and LSB reversed in a byte.
92 * @size must be integral times of unsigned long.
95 * f2fs_set_bit(0, bitmap) => 1000 0000
96 * f2fs_set_bit(7, bitmap) => 0000 0001
98 static unsigned long __find_rev_next_bit(const unsigned long *addr
,
99 unsigned long size
, unsigned long offset
)
101 const unsigned long *p
= addr
+ BIT_WORD(offset
);
102 unsigned long result
= size
;
108 size
-= (offset
& ~(BITS_PER_LONG
- 1));
109 offset
%= BITS_PER_LONG
;
115 tmp
= __reverse_ulong((unsigned char *)p
);
117 tmp
&= ~0UL >> offset
;
118 if (size
< BITS_PER_LONG
)
119 tmp
&= (~0UL << (BITS_PER_LONG
- size
));
123 if (size
<= BITS_PER_LONG
)
125 size
-= BITS_PER_LONG
;
131 return result
- size
+ __reverse_ffs(tmp
);
134 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr
,
135 unsigned long size
, unsigned long offset
)
137 const unsigned long *p
= addr
+ BIT_WORD(offset
);
138 unsigned long result
= size
;
144 size
-= (offset
& ~(BITS_PER_LONG
- 1));
145 offset
%= BITS_PER_LONG
;
151 tmp
= __reverse_ulong((unsigned char *)p
);
154 tmp
|= ~0UL << (BITS_PER_LONG
- offset
);
155 if (size
< BITS_PER_LONG
)
160 if (size
<= BITS_PER_LONG
)
162 size
-= BITS_PER_LONG
;
168 return result
- size
+ __reverse_ffz(tmp
);
171 bool f2fs_need_SSR(struct f2fs_sb_info
*sbi
)
173 int node_secs
= get_blocktype_secs(sbi
, F2FS_DIRTY_NODES
);
174 int dent_secs
= get_blocktype_secs(sbi
, F2FS_DIRTY_DENTS
);
175 int imeta_secs
= get_blocktype_secs(sbi
, F2FS_DIRTY_IMETA
);
177 if (f2fs_lfs_mode(sbi
))
179 if (sbi
->gc_mode
== GC_URGENT_HIGH
)
181 if (unlikely(is_sbi_flag_set(sbi
, SBI_CP_DISABLED
)))
184 return free_sections(sbi
) <= (node_secs
+ 2 * dent_secs
+ imeta_secs
+
185 SM_I(sbi
)->min_ssr_sections
+ reserved_sections(sbi
));
188 void f2fs_abort_atomic_write(struct inode
*inode
, bool clean
)
190 struct f2fs_inode_info
*fi
= F2FS_I(inode
);
192 if (!f2fs_is_atomic_file(inode
))
196 truncate_inode_pages_final(inode
->i_mapping
);
198 release_atomic_write_cnt(inode
);
199 clear_inode_flag(inode
, FI_ATOMIC_COMMITTED
);
200 clear_inode_flag(inode
, FI_ATOMIC_REPLACE
);
201 clear_inode_flag(inode
, FI_ATOMIC_FILE
);
202 stat_dec_atomic_inode(inode
);
204 F2FS_I(inode
)->atomic_write_task
= NULL
;
207 f2fs_i_size_write(inode
, fi
->original_i_size
);
208 fi
->original_i_size
= 0;
210 /* avoid stale dirty inode during eviction */
211 sync_inode_metadata(inode
, 0);
214 static int __replace_atomic_write_block(struct inode
*inode
, pgoff_t index
,
215 block_t new_addr
, block_t
*old_addr
, bool recover
)
217 struct f2fs_sb_info
*sbi
= F2FS_I_SB(inode
);
218 struct dnode_of_data dn
;
223 set_new_dnode(&dn
, inode
, NULL
, NULL
, 0);
224 err
= f2fs_get_dnode_of_data(&dn
, index
, ALLOC_NODE
);
226 if (err
== -ENOMEM
) {
227 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT
);
233 err
= f2fs_get_node_info(sbi
, dn
.nid
, &ni
, false);
240 /* dn.data_blkaddr is always valid */
241 if (!__is_valid_data_blkaddr(new_addr
)) {
242 if (new_addr
== NULL_ADDR
)
243 dec_valid_block_count(sbi
, inode
, 1);
244 f2fs_invalidate_blocks(sbi
, dn
.data_blkaddr
);
245 f2fs_update_data_blkaddr(&dn
, new_addr
);
247 f2fs_replace_block(sbi
, &dn
, dn
.data_blkaddr
,
248 new_addr
, ni
.version
, true, true);
253 err
= inc_valid_block_count(sbi
, inode
, &count
, true);
259 *old_addr
= dn
.data_blkaddr
;
260 f2fs_truncate_data_blocks_range(&dn
, 1);
261 dec_valid_block_count(sbi
, F2FS_I(inode
)->cow_inode
, count
);
263 f2fs_replace_block(sbi
, &dn
, dn
.data_blkaddr
, new_addr
,
264 ni
.version
, true, false);
269 trace_f2fs_replace_atomic_write_block(inode
, F2FS_I(inode
)->cow_inode
,
270 index
, old_addr
? *old_addr
: 0, new_addr
, recover
);
274 static void __complete_revoke_list(struct inode
*inode
, struct list_head
*head
,
277 struct revoke_entry
*cur
, *tmp
;
278 pgoff_t start_index
= 0;
279 bool truncate
= is_inode_flag_set(inode
, FI_ATOMIC_REPLACE
);
281 list_for_each_entry_safe(cur
, tmp
, head
, list
) {
283 __replace_atomic_write_block(inode
, cur
->index
,
284 cur
->old_addr
, NULL
, true);
285 } else if (truncate
) {
286 f2fs_truncate_hole(inode
, start_index
, cur
->index
);
287 start_index
= cur
->index
+ 1;
290 list_del(&cur
->list
);
291 kmem_cache_free(revoke_entry_slab
, cur
);
294 if (!revoke
&& truncate
)
295 f2fs_do_truncate_blocks(inode
, start_index
* PAGE_SIZE
, false);
298 static int __f2fs_commit_atomic_write(struct inode
*inode
)
300 struct f2fs_sb_info
*sbi
= F2FS_I_SB(inode
);
301 struct f2fs_inode_info
*fi
= F2FS_I(inode
);
302 struct inode
*cow_inode
= fi
->cow_inode
;
303 struct revoke_entry
*new;
304 struct list_head revoke_list
;
306 struct dnode_of_data dn
;
307 pgoff_t len
= DIV_ROUND_UP(i_size_read(inode
), PAGE_SIZE
);
308 pgoff_t off
= 0, blen
, index
;
311 INIT_LIST_HEAD(&revoke_list
);
314 blen
= min_t(pgoff_t
, ADDRS_PER_BLOCK(cow_inode
), len
);
316 set_new_dnode(&dn
, cow_inode
, NULL
, NULL
, 0);
317 ret
= f2fs_get_dnode_of_data(&dn
, off
, LOOKUP_NODE_RA
);
318 if (ret
&& ret
!= -ENOENT
) {
320 } else if (ret
== -ENOENT
) {
322 if (dn
.max_level
== 0)
327 blen
= min((pgoff_t
)ADDRS_PER_PAGE(dn
.node_page
, cow_inode
),
330 for (i
= 0; i
< blen
; i
++, dn
.ofs_in_node
++, index
++) {
331 blkaddr
= f2fs_data_blkaddr(&dn
);
333 if (!__is_valid_data_blkaddr(blkaddr
)) {
335 } else if (!f2fs_is_valid_blkaddr(sbi
, blkaddr
,
336 DATA_GENERIC_ENHANCE
)) {
342 new = f2fs_kmem_cache_alloc(revoke_entry_slab
, GFP_NOFS
,
345 ret
= __replace_atomic_write_block(inode
, index
, blkaddr
,
346 &new->old_addr
, false);
349 kmem_cache_free(revoke_entry_slab
, new);
353 f2fs_update_data_blkaddr(&dn
, NULL_ADDR
);
355 list_add_tail(&new->list
, &revoke_list
);
365 sbi
->revoked_atomic_block
+= fi
->atomic_write_cnt
;
367 sbi
->committed_atomic_block
+= fi
->atomic_write_cnt
;
368 set_inode_flag(inode
, FI_ATOMIC_COMMITTED
);
371 __complete_revoke_list(inode
, &revoke_list
, ret
? true : false);
376 int f2fs_commit_atomic_write(struct inode
*inode
)
378 struct f2fs_sb_info
*sbi
= F2FS_I_SB(inode
);
379 struct f2fs_inode_info
*fi
= F2FS_I(inode
);
382 err
= filemap_write_and_wait_range(inode
->i_mapping
, 0, LLONG_MAX
);
386 f2fs_down_write(&fi
->i_gc_rwsem
[WRITE
]);
389 err
= __f2fs_commit_atomic_write(inode
);
392 f2fs_up_write(&fi
->i_gc_rwsem
[WRITE
]);
398 * This function balances dirty node and dentry pages.
399 * In addition, it controls garbage collection.
401 void f2fs_balance_fs(struct f2fs_sb_info
*sbi
, bool need
)
403 if (f2fs_cp_error(sbi
))
406 if (time_to_inject(sbi
, FAULT_CHECKPOINT
))
407 f2fs_stop_checkpoint(sbi
, false, STOP_CP_REASON_FAULT_INJECT
);
409 /* balance_fs_bg is able to be pending */
410 if (need
&& excess_cached_nats(sbi
))
411 f2fs_balance_fs_bg(sbi
, false);
413 if (!f2fs_is_checkpoint_ready(sbi
))
417 * We should do GC or end up with checkpoint, if there are so many dirty
418 * dir/node pages without enough free segments.
420 if (has_enough_free_secs(sbi
, 0, 0))
423 if (test_opt(sbi
, GC_MERGE
) && sbi
->gc_thread
&&
424 sbi
->gc_thread
->f2fs_gc_task
) {
427 prepare_to_wait(&sbi
->gc_thread
->fggc_wq
, &wait
,
428 TASK_UNINTERRUPTIBLE
);
429 wake_up(&sbi
->gc_thread
->gc_wait_queue_head
);
431 finish_wait(&sbi
->gc_thread
->fggc_wq
, &wait
);
433 struct f2fs_gc_control gc_control
= {
434 .victim_segno
= NULL_SEGNO
,
435 .init_gc_type
= BG_GC
,
437 .should_migrate_blocks
= false,
438 .err_gc_skipped
= false,
440 f2fs_down_write(&sbi
->gc_lock
);
441 stat_inc_gc_call_count(sbi
, FOREGROUND
);
442 f2fs_gc(sbi
, &gc_control
);
446 static inline bool excess_dirty_threshold(struct f2fs_sb_info
*sbi
)
448 int factor
= f2fs_rwsem_is_locked(&sbi
->cp_rwsem
) ? 3 : 2;
449 unsigned int dents
= get_pages(sbi
, F2FS_DIRTY_DENTS
);
450 unsigned int qdata
= get_pages(sbi
, F2FS_DIRTY_QDATA
);
451 unsigned int nodes
= get_pages(sbi
, F2FS_DIRTY_NODES
);
452 unsigned int meta
= get_pages(sbi
, F2FS_DIRTY_META
);
453 unsigned int imeta
= get_pages(sbi
, F2FS_DIRTY_IMETA
);
454 unsigned int threshold
=
455 SEGS_TO_BLKS(sbi
, (factor
* DEFAULT_DIRTY_THRESHOLD
));
456 unsigned int global_threshold
= threshold
* 3 / 2;
458 if (dents
>= threshold
|| qdata
>= threshold
||
459 nodes
>= threshold
|| meta
>= threshold
||
462 return dents
+ qdata
+ nodes
+ meta
+ imeta
> global_threshold
;
465 void f2fs_balance_fs_bg(struct f2fs_sb_info
*sbi
, bool from_bg
)
467 if (unlikely(is_sbi_flag_set(sbi
, SBI_POR_DOING
)))
470 /* try to shrink extent cache when there is no enough memory */
471 if (!f2fs_available_free_memory(sbi
, READ_EXTENT_CACHE
))
472 f2fs_shrink_read_extent_tree(sbi
,
473 READ_EXTENT_CACHE_SHRINK_NUMBER
);
475 /* try to shrink age extent cache when there is no enough memory */
476 if (!f2fs_available_free_memory(sbi
, AGE_EXTENT_CACHE
))
477 f2fs_shrink_age_extent_tree(sbi
,
478 AGE_EXTENT_CACHE_SHRINK_NUMBER
);
480 /* check the # of cached NAT entries */
481 if (!f2fs_available_free_memory(sbi
, NAT_ENTRIES
))
482 f2fs_try_to_free_nats(sbi
, NAT_ENTRY_PER_BLOCK
);
484 if (!f2fs_available_free_memory(sbi
, FREE_NIDS
))
485 f2fs_try_to_free_nids(sbi
, MAX_FREE_NIDS
);
487 f2fs_build_free_nids(sbi
, false, false);
489 if (excess_dirty_nats(sbi
) || excess_dirty_threshold(sbi
) ||
490 excess_prefree_segs(sbi
) || !f2fs_space_for_roll_forward(sbi
))
493 /* there is background inflight IO or foreground operation recently */
494 if (is_inflight_io(sbi
, REQ_TIME
) ||
495 (!f2fs_time_over(sbi
, REQ_TIME
) && f2fs_rwsem_is_locked(&sbi
->cp_rwsem
)))
498 /* exceed periodical checkpoint timeout threshold */
499 if (f2fs_time_over(sbi
, CP_TIME
))
502 /* checkpoint is the only way to shrink partial cached entries */
503 if (f2fs_available_free_memory(sbi
, NAT_ENTRIES
) &&
504 f2fs_available_free_memory(sbi
, INO_ENTRIES
))
508 if (test_opt(sbi
, DATA_FLUSH
) && from_bg
) {
509 struct blk_plug plug
;
511 mutex_lock(&sbi
->flush_lock
);
513 blk_start_plug(&plug
);
514 f2fs_sync_dirty_inodes(sbi
, FILE_INODE
, false);
515 blk_finish_plug(&plug
);
517 mutex_unlock(&sbi
->flush_lock
);
519 stat_inc_cp_call_count(sbi
, BACKGROUND
);
520 f2fs_sync_fs(sbi
->sb
, 1);
523 static int __submit_flush_wait(struct f2fs_sb_info
*sbi
,
524 struct block_device
*bdev
)
526 int ret
= blkdev_issue_flush(bdev
);
528 trace_f2fs_issue_flush(bdev
, test_opt(sbi
, NOBARRIER
),
529 test_opt(sbi
, FLUSH_MERGE
), ret
);
531 f2fs_update_iostat(sbi
, NULL
, FS_FLUSH_IO
, 0);
535 static int submit_flush_wait(struct f2fs_sb_info
*sbi
, nid_t ino
)
540 if (!f2fs_is_multi_device(sbi
))
541 return __submit_flush_wait(sbi
, sbi
->sb
->s_bdev
);
543 for (i
= 0; i
< sbi
->s_ndevs
; i
++) {
544 if (!f2fs_is_dirty_device(sbi
, ino
, i
, FLUSH_INO
))
546 ret
= __submit_flush_wait(sbi
, FDEV(i
).bdev
);
553 static int issue_flush_thread(void *data
)
555 struct f2fs_sb_info
*sbi
= data
;
556 struct flush_cmd_control
*fcc
= SM_I(sbi
)->fcc_info
;
557 wait_queue_head_t
*q
= &fcc
->flush_wait_queue
;
559 if (kthread_should_stop())
562 if (!llist_empty(&fcc
->issue_list
)) {
563 struct flush_cmd
*cmd
, *next
;
566 fcc
->dispatch_list
= llist_del_all(&fcc
->issue_list
);
567 fcc
->dispatch_list
= llist_reverse_order(fcc
->dispatch_list
);
569 cmd
= llist_entry(fcc
->dispatch_list
, struct flush_cmd
, llnode
);
571 ret
= submit_flush_wait(sbi
, cmd
->ino
);
572 atomic_inc(&fcc
->issued_flush
);
574 llist_for_each_entry_safe(cmd
, next
,
575 fcc
->dispatch_list
, llnode
) {
577 complete(&cmd
->wait
);
579 fcc
->dispatch_list
= NULL
;
582 wait_event_interruptible(*q
,
583 kthread_should_stop() || !llist_empty(&fcc
->issue_list
));
587 int f2fs_issue_flush(struct f2fs_sb_info
*sbi
, nid_t ino
)
589 struct flush_cmd_control
*fcc
= SM_I(sbi
)->fcc_info
;
590 struct flush_cmd cmd
;
593 if (test_opt(sbi
, NOBARRIER
))
596 if (!test_opt(sbi
, FLUSH_MERGE
)) {
597 atomic_inc(&fcc
->queued_flush
);
598 ret
= submit_flush_wait(sbi
, ino
);
599 atomic_dec(&fcc
->queued_flush
);
600 atomic_inc(&fcc
->issued_flush
);
604 if (atomic_inc_return(&fcc
->queued_flush
) == 1 ||
605 f2fs_is_multi_device(sbi
)) {
606 ret
= submit_flush_wait(sbi
, ino
);
607 atomic_dec(&fcc
->queued_flush
);
609 atomic_inc(&fcc
->issued_flush
);
614 init_completion(&cmd
.wait
);
616 llist_add(&cmd
.llnode
, &fcc
->issue_list
);
619 * update issue_list before we wake up issue_flush thread, this
620 * smp_mb() pairs with another barrier in ___wait_event(), see
621 * more details in comments of waitqueue_active().
625 if (waitqueue_active(&fcc
->flush_wait_queue
))
626 wake_up(&fcc
->flush_wait_queue
);
628 if (fcc
->f2fs_issue_flush
) {
629 wait_for_completion(&cmd
.wait
);
630 atomic_dec(&fcc
->queued_flush
);
632 struct llist_node
*list
;
634 list
= llist_del_all(&fcc
->issue_list
);
636 wait_for_completion(&cmd
.wait
);
637 atomic_dec(&fcc
->queued_flush
);
639 struct flush_cmd
*tmp
, *next
;
641 ret
= submit_flush_wait(sbi
, ino
);
643 llist_for_each_entry_safe(tmp
, next
, list
, llnode
) {
646 atomic_dec(&fcc
->queued_flush
);
650 complete(&tmp
->wait
);
658 int f2fs_create_flush_cmd_control(struct f2fs_sb_info
*sbi
)
660 dev_t dev
= sbi
->sb
->s_bdev
->bd_dev
;
661 struct flush_cmd_control
*fcc
;
663 if (SM_I(sbi
)->fcc_info
) {
664 fcc
= SM_I(sbi
)->fcc_info
;
665 if (fcc
->f2fs_issue_flush
)
670 fcc
= f2fs_kzalloc(sbi
, sizeof(struct flush_cmd_control
), GFP_KERNEL
);
673 atomic_set(&fcc
->issued_flush
, 0);
674 atomic_set(&fcc
->queued_flush
, 0);
675 init_waitqueue_head(&fcc
->flush_wait_queue
);
676 init_llist_head(&fcc
->issue_list
);
677 SM_I(sbi
)->fcc_info
= fcc
;
678 if (!test_opt(sbi
, FLUSH_MERGE
))
682 fcc
->f2fs_issue_flush
= kthread_run(issue_flush_thread
, sbi
,
683 "f2fs_flush-%u:%u", MAJOR(dev
), MINOR(dev
));
684 if (IS_ERR(fcc
->f2fs_issue_flush
)) {
685 int err
= PTR_ERR(fcc
->f2fs_issue_flush
);
687 fcc
->f2fs_issue_flush
= NULL
;
694 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info
*sbi
, bool free
)
696 struct flush_cmd_control
*fcc
= SM_I(sbi
)->fcc_info
;
698 if (fcc
&& fcc
->f2fs_issue_flush
) {
699 struct task_struct
*flush_thread
= fcc
->f2fs_issue_flush
;
701 fcc
->f2fs_issue_flush
= NULL
;
702 kthread_stop(flush_thread
);
706 SM_I(sbi
)->fcc_info
= NULL
;
710 int f2fs_flush_device_cache(struct f2fs_sb_info
*sbi
)
714 if (!f2fs_is_multi_device(sbi
))
717 if (test_opt(sbi
, NOBARRIER
))
720 for (i
= 1; i
< sbi
->s_ndevs
; i
++) {
721 int count
= DEFAULT_RETRY_IO_COUNT
;
723 if (!f2fs_test_bit(i
, (char *)&sbi
->dirty_device
))
727 ret
= __submit_flush_wait(sbi
, FDEV(i
).bdev
);
729 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT
);
730 } while (ret
&& --count
);
733 f2fs_stop_checkpoint(sbi
, false,
734 STOP_CP_REASON_FLUSH_FAIL
);
738 spin_lock(&sbi
->dev_lock
);
739 f2fs_clear_bit(i
, (char *)&sbi
->dirty_device
);
740 spin_unlock(&sbi
->dev_lock
);
746 static void __locate_dirty_segment(struct f2fs_sb_info
*sbi
, unsigned int segno
,
747 enum dirty_type dirty_type
)
749 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
751 /* need not be added */
752 if (IS_CURSEG(sbi
, segno
))
755 if (!test_and_set_bit(segno
, dirty_i
->dirty_segmap
[dirty_type
]))
756 dirty_i
->nr_dirty
[dirty_type
]++;
758 if (dirty_type
== DIRTY
) {
759 struct seg_entry
*sentry
= get_seg_entry(sbi
, segno
);
760 enum dirty_type t
= sentry
->type
;
762 if (unlikely(t
>= DIRTY
)) {
766 if (!test_and_set_bit(segno
, dirty_i
->dirty_segmap
[t
]))
767 dirty_i
->nr_dirty
[t
]++;
769 if (__is_large_section(sbi
)) {
770 unsigned int secno
= GET_SEC_FROM_SEG(sbi
, segno
);
771 block_t valid_blocks
=
772 get_valid_blocks(sbi
, segno
, true);
775 (!is_sbi_flag_set(sbi
, SBI_CP_DISABLED
) &&
777 valid_blocks
== CAP_BLKS_PER_SEC(sbi
));
779 if (!IS_CURSEC(sbi
, secno
))
780 set_bit(secno
, dirty_i
->dirty_secmap
);
785 static void __remove_dirty_segment(struct f2fs_sb_info
*sbi
, unsigned int segno
,
786 enum dirty_type dirty_type
)
788 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
789 block_t valid_blocks
;
791 if (test_and_clear_bit(segno
, dirty_i
->dirty_segmap
[dirty_type
]))
792 dirty_i
->nr_dirty
[dirty_type
]--;
794 if (dirty_type
== DIRTY
) {
795 struct seg_entry
*sentry
= get_seg_entry(sbi
, segno
);
796 enum dirty_type t
= sentry
->type
;
798 if (test_and_clear_bit(segno
, dirty_i
->dirty_segmap
[t
]))
799 dirty_i
->nr_dirty
[t
]--;
801 valid_blocks
= get_valid_blocks(sbi
, segno
, true);
802 if (valid_blocks
== 0) {
803 clear_bit(GET_SEC_FROM_SEG(sbi
, segno
),
804 dirty_i
->victim_secmap
);
805 #ifdef CONFIG_F2FS_CHECK_FS
806 clear_bit(segno
, SIT_I(sbi
)->invalid_segmap
);
809 if (__is_large_section(sbi
)) {
810 unsigned int secno
= GET_SEC_FROM_SEG(sbi
, segno
);
813 valid_blocks
== CAP_BLKS_PER_SEC(sbi
)) {
814 clear_bit(secno
, dirty_i
->dirty_secmap
);
818 if (!IS_CURSEC(sbi
, secno
))
819 set_bit(secno
, dirty_i
->dirty_secmap
);
825 * Should not occur error such as -ENOMEM.
826 * Adding dirty entry into seglist is not critical operation.
827 * If a given segment is one of current working segments, it won't be added.
829 static void locate_dirty_segment(struct f2fs_sb_info
*sbi
, unsigned int segno
)
831 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
832 unsigned short valid_blocks
, ckpt_valid_blocks
;
833 unsigned int usable_blocks
;
835 if (segno
== NULL_SEGNO
|| IS_CURSEG(sbi
, segno
))
838 usable_blocks
= f2fs_usable_blks_in_seg(sbi
, segno
);
839 mutex_lock(&dirty_i
->seglist_lock
);
841 valid_blocks
= get_valid_blocks(sbi
, segno
, false);
842 ckpt_valid_blocks
= get_ckpt_valid_blocks(sbi
, segno
, false);
844 if (valid_blocks
== 0 && (!is_sbi_flag_set(sbi
, SBI_CP_DISABLED
) ||
845 ckpt_valid_blocks
== usable_blocks
)) {
846 __locate_dirty_segment(sbi
, segno
, PRE
);
847 __remove_dirty_segment(sbi
, segno
, DIRTY
);
848 } else if (valid_blocks
< usable_blocks
) {
849 __locate_dirty_segment(sbi
, segno
, DIRTY
);
851 /* Recovery routine with SSR needs this */
852 __remove_dirty_segment(sbi
, segno
, DIRTY
);
855 mutex_unlock(&dirty_i
->seglist_lock
);
858 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
859 void f2fs_dirty_to_prefree(struct f2fs_sb_info
*sbi
)
861 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
864 mutex_lock(&dirty_i
->seglist_lock
);
865 for_each_set_bit(segno
, dirty_i
->dirty_segmap
[DIRTY
], MAIN_SEGS(sbi
)) {
866 if (get_valid_blocks(sbi
, segno
, false))
868 if (IS_CURSEG(sbi
, segno
))
870 __locate_dirty_segment(sbi
, segno
, PRE
);
871 __remove_dirty_segment(sbi
, segno
, DIRTY
);
873 mutex_unlock(&dirty_i
->seglist_lock
);
876 block_t
f2fs_get_unusable_blocks(struct f2fs_sb_info
*sbi
)
879 (overprovision_segments(sbi
) - reserved_segments(sbi
));
880 block_t ovp_holes
= SEGS_TO_BLKS(sbi
, ovp_hole_segs
);
881 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
882 block_t holes
[2] = {0, 0}; /* DATA and NODE */
884 struct seg_entry
*se
;
887 mutex_lock(&dirty_i
->seglist_lock
);
888 for_each_set_bit(segno
, dirty_i
->dirty_segmap
[DIRTY
], MAIN_SEGS(sbi
)) {
889 se
= get_seg_entry(sbi
, segno
);
890 if (IS_NODESEG(se
->type
))
891 holes
[NODE
] += f2fs_usable_blks_in_seg(sbi
, segno
) -
894 holes
[DATA
] += f2fs_usable_blks_in_seg(sbi
, segno
) -
897 mutex_unlock(&dirty_i
->seglist_lock
);
899 unusable
= max(holes
[DATA
], holes
[NODE
]);
900 if (unusable
> ovp_holes
)
901 return unusable
- ovp_holes
;
905 int f2fs_disable_cp_again(struct f2fs_sb_info
*sbi
, block_t unusable
)
908 (overprovision_segments(sbi
) - reserved_segments(sbi
));
910 if (F2FS_OPTION(sbi
).unusable_cap_perc
== 100)
912 if (unusable
> F2FS_OPTION(sbi
).unusable_cap
)
914 if (is_sbi_flag_set(sbi
, SBI_CP_DISABLED_QUICK
) &&
915 dirty_segments(sbi
) > ovp_hole_segs
)
917 if (has_not_enough_free_secs(sbi
, 0, 0))
922 /* This is only used by SBI_CP_DISABLED */
923 static unsigned int get_free_segment(struct f2fs_sb_info
*sbi
)
925 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
926 unsigned int segno
= 0;
928 mutex_lock(&dirty_i
->seglist_lock
);
929 for_each_set_bit(segno
, dirty_i
->dirty_segmap
[DIRTY
], MAIN_SEGS(sbi
)) {
930 if (get_valid_blocks(sbi
, segno
, false))
932 if (get_ckpt_valid_blocks(sbi
, segno
, false))
934 mutex_unlock(&dirty_i
->seglist_lock
);
937 mutex_unlock(&dirty_i
->seglist_lock
);
941 static struct discard_cmd
*__create_discard_cmd(struct f2fs_sb_info
*sbi
,
942 struct block_device
*bdev
, block_t lstart
,
943 block_t start
, block_t len
)
945 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
946 struct list_head
*pend_list
;
947 struct discard_cmd
*dc
;
949 f2fs_bug_on(sbi
, !len
);
951 pend_list
= &dcc
->pend_list
[plist_idx(len
)];
953 dc
= f2fs_kmem_cache_alloc(discard_cmd_slab
, GFP_NOFS
, true, NULL
);
954 INIT_LIST_HEAD(&dc
->list
);
956 dc
->di
.lstart
= lstart
;
957 dc
->di
.start
= start
;
963 init_completion(&dc
->wait
);
964 list_add_tail(&dc
->list
, pend_list
);
965 spin_lock_init(&dc
->lock
);
967 atomic_inc(&dcc
->discard_cmd_cnt
);
968 dcc
->undiscard_blks
+= len
;
973 static bool f2fs_check_discard_tree(struct f2fs_sb_info
*sbi
)
975 #ifdef CONFIG_F2FS_CHECK_FS
976 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
977 struct rb_node
*cur
= rb_first_cached(&dcc
->root
), *next
;
978 struct discard_cmd
*cur_dc
, *next_dc
;
985 cur_dc
= rb_entry(cur
, struct discard_cmd
, rb_node
);
986 next_dc
= rb_entry(next
, struct discard_cmd
, rb_node
);
988 if (cur_dc
->di
.lstart
+ cur_dc
->di
.len
> next_dc
->di
.lstart
) {
989 f2fs_info(sbi
, "broken discard_rbtree, "
990 "cur(%u, %u) next(%u, %u)",
991 cur_dc
->di
.lstart
, cur_dc
->di
.len
,
992 next_dc
->di
.lstart
, next_dc
->di
.len
);
1001 static struct discard_cmd
*__lookup_discard_cmd(struct f2fs_sb_info
*sbi
,
1004 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1005 struct rb_node
*node
= dcc
->root
.rb_root
.rb_node
;
1006 struct discard_cmd
*dc
;
1009 dc
= rb_entry(node
, struct discard_cmd
, rb_node
);
1011 if (blkaddr
< dc
->di
.lstart
)
1012 node
= node
->rb_left
;
1013 else if (blkaddr
>= dc
->di
.lstart
+ dc
->di
.len
)
1014 node
= node
->rb_right
;
1021 static struct discard_cmd
*__lookup_discard_cmd_ret(struct rb_root_cached
*root
,
1023 struct discard_cmd
**prev_entry
,
1024 struct discard_cmd
**next_entry
,
1025 struct rb_node
***insert_p
,
1026 struct rb_node
**insert_parent
)
1028 struct rb_node
**pnode
= &root
->rb_root
.rb_node
;
1029 struct rb_node
*parent
= NULL
, *tmp_node
;
1030 struct discard_cmd
*dc
;
1033 *insert_parent
= NULL
;
1037 if (RB_EMPTY_ROOT(&root
->rb_root
))
1042 dc
= rb_entry(*pnode
, struct discard_cmd
, rb_node
);
1044 if (blkaddr
< dc
->di
.lstart
)
1045 pnode
= &(*pnode
)->rb_left
;
1046 else if (blkaddr
>= dc
->di
.lstart
+ dc
->di
.len
)
1047 pnode
= &(*pnode
)->rb_right
;
1049 goto lookup_neighbors
;
1053 *insert_parent
= parent
;
1055 dc
= rb_entry(parent
, struct discard_cmd
, rb_node
);
1057 if (parent
&& blkaddr
> dc
->di
.lstart
)
1058 tmp_node
= rb_next(parent
);
1059 *next_entry
= rb_entry_safe(tmp_node
, struct discard_cmd
, rb_node
);
1062 if (parent
&& blkaddr
< dc
->di
.lstart
)
1063 tmp_node
= rb_prev(parent
);
1064 *prev_entry
= rb_entry_safe(tmp_node
, struct discard_cmd
, rb_node
);
1068 /* lookup prev node for merging backward later */
1069 tmp_node
= rb_prev(&dc
->rb_node
);
1070 *prev_entry
= rb_entry_safe(tmp_node
, struct discard_cmd
, rb_node
);
1072 /* lookup next node for merging frontward later */
1073 tmp_node
= rb_next(&dc
->rb_node
);
1074 *next_entry
= rb_entry_safe(tmp_node
, struct discard_cmd
, rb_node
);
1078 static void __detach_discard_cmd(struct discard_cmd_control
*dcc
,
1079 struct discard_cmd
*dc
)
1081 if (dc
->state
== D_DONE
)
1082 atomic_sub(dc
->queued
, &dcc
->queued_discard
);
1084 list_del(&dc
->list
);
1085 rb_erase_cached(&dc
->rb_node
, &dcc
->root
);
1086 dcc
->undiscard_blks
-= dc
->di
.len
;
1088 kmem_cache_free(discard_cmd_slab
, dc
);
1090 atomic_dec(&dcc
->discard_cmd_cnt
);
1093 static void __remove_discard_cmd(struct f2fs_sb_info
*sbi
,
1094 struct discard_cmd
*dc
)
1096 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1097 unsigned long flags
;
1099 trace_f2fs_remove_discard(dc
->bdev
, dc
->di
.start
, dc
->di
.len
);
1101 spin_lock_irqsave(&dc
->lock
, flags
);
1103 spin_unlock_irqrestore(&dc
->lock
, flags
);
1106 spin_unlock_irqrestore(&dc
->lock
, flags
);
1108 f2fs_bug_on(sbi
, dc
->ref
);
1110 if (dc
->error
== -EOPNOTSUPP
)
1114 f2fs_info_ratelimited(sbi
,
1115 "Issue discard(%u, %u, %u) failed, ret: %d",
1116 dc
->di
.lstart
, dc
->di
.start
, dc
->di
.len
, dc
->error
);
1117 __detach_discard_cmd(dcc
, dc
);
1120 static void f2fs_submit_discard_endio(struct bio
*bio
)
1122 struct discard_cmd
*dc
= (struct discard_cmd
*)bio
->bi_private
;
1123 unsigned long flags
;
1125 spin_lock_irqsave(&dc
->lock
, flags
);
1127 dc
->error
= blk_status_to_errno(bio
->bi_status
);
1129 if (!dc
->bio_ref
&& dc
->state
== D_SUBMIT
) {
1131 complete_all(&dc
->wait
);
1133 spin_unlock_irqrestore(&dc
->lock
, flags
);
1137 static void __check_sit_bitmap(struct f2fs_sb_info
*sbi
,
1138 block_t start
, block_t end
)
1140 #ifdef CONFIG_F2FS_CHECK_FS
1141 struct seg_entry
*sentry
;
1143 block_t blk
= start
;
1144 unsigned long offset
, size
, *map
;
1147 segno
= GET_SEGNO(sbi
, blk
);
1148 sentry
= get_seg_entry(sbi
, segno
);
1149 offset
= GET_BLKOFF_FROM_SEG0(sbi
, blk
);
1151 if (end
< START_BLOCK(sbi
, segno
+ 1))
1152 size
= GET_BLKOFF_FROM_SEG0(sbi
, end
);
1154 size
= BLKS_PER_SEG(sbi
);
1155 map
= (unsigned long *)(sentry
->cur_valid_map
);
1156 offset
= __find_rev_next_bit(map
, size
, offset
);
1157 f2fs_bug_on(sbi
, offset
!= size
);
1158 blk
= START_BLOCK(sbi
, segno
+ 1);
1163 static void __init_discard_policy(struct f2fs_sb_info
*sbi
,
1164 struct discard_policy
*dpolicy
,
1165 int discard_type
, unsigned int granularity
)
1167 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1170 dpolicy
->type
= discard_type
;
1171 dpolicy
->sync
= true;
1172 dpolicy
->ordered
= false;
1173 dpolicy
->granularity
= granularity
;
1175 dpolicy
->max_requests
= dcc
->max_discard_request
;
1176 dpolicy
->io_aware_gran
= dcc
->discard_io_aware_gran
;
1177 dpolicy
->timeout
= false;
1179 if (discard_type
== DPOLICY_BG
) {
1180 dpolicy
->min_interval
= dcc
->min_discard_issue_time
;
1181 dpolicy
->mid_interval
= dcc
->mid_discard_issue_time
;
1182 dpolicy
->max_interval
= dcc
->max_discard_issue_time
;
1183 if (dcc
->discard_io_aware
== DPOLICY_IO_AWARE_ENABLE
)
1184 dpolicy
->io_aware
= true;
1185 else if (dcc
->discard_io_aware
== DPOLICY_IO_AWARE_DISABLE
)
1186 dpolicy
->io_aware
= false;
1187 dpolicy
->sync
= false;
1188 dpolicy
->ordered
= true;
1189 if (utilization(sbi
) > dcc
->discard_urgent_util
) {
1190 dpolicy
->granularity
= MIN_DISCARD_GRANULARITY
;
1191 if (atomic_read(&dcc
->discard_cmd_cnt
))
1192 dpolicy
->max_interval
=
1193 dcc
->min_discard_issue_time
;
1195 } else if (discard_type
== DPOLICY_FORCE
) {
1196 dpolicy
->min_interval
= dcc
->min_discard_issue_time
;
1197 dpolicy
->mid_interval
= dcc
->mid_discard_issue_time
;
1198 dpolicy
->max_interval
= dcc
->max_discard_issue_time
;
1199 dpolicy
->io_aware
= false;
1200 } else if (discard_type
== DPOLICY_FSTRIM
) {
1201 dpolicy
->io_aware
= false;
1202 } else if (discard_type
== DPOLICY_UMOUNT
) {
1203 dpolicy
->io_aware
= false;
1204 /* we need to issue all to keep CP_TRIMMED_FLAG */
1205 dpolicy
->granularity
= MIN_DISCARD_GRANULARITY
;
1206 dpolicy
->timeout
= true;
1210 static void __update_discard_tree_range(struct f2fs_sb_info
*sbi
,
1211 struct block_device
*bdev
, block_t lstart
,
1212 block_t start
, block_t len
);
1214 #ifdef CONFIG_BLK_DEV_ZONED
1215 static void __submit_zone_reset_cmd(struct f2fs_sb_info
*sbi
,
1216 struct discard_cmd
*dc
, blk_opf_t flag
,
1217 struct list_head
*wait_list
,
1218 unsigned int *issued
)
1220 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1221 struct block_device
*bdev
= dc
->bdev
;
1222 struct bio
*bio
= bio_alloc(bdev
, 0, REQ_OP_ZONE_RESET
| flag
, GFP_NOFS
);
1223 unsigned long flags
;
1225 trace_f2fs_issue_reset_zone(bdev
, dc
->di
.start
);
1227 spin_lock_irqsave(&dc
->lock
, flags
);
1228 dc
->state
= D_SUBMIT
;
1230 spin_unlock_irqrestore(&dc
->lock
, flags
);
1235 atomic_inc(&dcc
->queued_discard
);
1237 list_move_tail(&dc
->list
, wait_list
);
1239 /* sanity check on discard range */
1240 __check_sit_bitmap(sbi
, dc
->di
.lstart
, dc
->di
.lstart
+ dc
->di
.len
);
1242 bio
->bi_iter
.bi_sector
= SECTOR_FROM_BLOCK(dc
->di
.start
);
1243 bio
->bi_private
= dc
;
1244 bio
->bi_end_io
= f2fs_submit_discard_endio
;
1247 atomic_inc(&dcc
->issued_discard
);
1248 f2fs_update_iostat(sbi
, NULL
, FS_ZONE_RESET_IO
, dc
->di
.len
* F2FS_BLKSIZE
);
1252 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
1253 static int __submit_discard_cmd(struct f2fs_sb_info
*sbi
,
1254 struct discard_policy
*dpolicy
,
1255 struct discard_cmd
*dc
, int *issued
)
1257 struct block_device
*bdev
= dc
->bdev
;
1258 unsigned int max_discard_blocks
=
1259 SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev
));
1260 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1261 struct list_head
*wait_list
= (dpolicy
->type
== DPOLICY_FSTRIM
) ?
1262 &(dcc
->fstrim_list
) : &(dcc
->wait_list
);
1263 blk_opf_t flag
= dpolicy
->sync
? REQ_SYNC
: 0;
1264 block_t lstart
, start
, len
, total_len
;
1267 if (dc
->state
!= D_PREP
)
1270 if (is_sbi_flag_set(sbi
, SBI_NEED_FSCK
))
1273 #ifdef CONFIG_BLK_DEV_ZONED
1274 if (f2fs_sb_has_blkzoned(sbi
) && bdev_is_zoned(bdev
)) {
1275 int devi
= f2fs_bdev_index(sbi
, bdev
);
1280 if (f2fs_blkz_is_seq(sbi
, devi
, dc
->di
.start
)) {
1281 __submit_zone_reset_cmd(sbi
, dc
, flag
,
1288 trace_f2fs_issue_discard(bdev
, dc
->di
.start
, dc
->di
.len
);
1290 lstart
= dc
->di
.lstart
;
1291 start
= dc
->di
.start
;
1297 while (total_len
&& *issued
< dpolicy
->max_requests
&& !err
) {
1298 struct bio
*bio
= NULL
;
1299 unsigned long flags
;
1302 if (len
> max_discard_blocks
) {
1303 len
= max_discard_blocks
;
1308 if (*issued
== dpolicy
->max_requests
)
1313 if (time_to_inject(sbi
, FAULT_DISCARD
)) {
1316 err
= __blkdev_issue_discard(bdev
,
1317 SECTOR_FROM_BLOCK(start
),
1318 SECTOR_FROM_BLOCK(len
),
1322 spin_lock_irqsave(&dc
->lock
, flags
);
1323 if (dc
->state
== D_PARTIAL
)
1324 dc
->state
= D_SUBMIT
;
1325 spin_unlock_irqrestore(&dc
->lock
, flags
);
1330 f2fs_bug_on(sbi
, !bio
);
1333 * should keep before submission to avoid D_DONE
1336 spin_lock_irqsave(&dc
->lock
, flags
);
1338 dc
->state
= D_SUBMIT
;
1340 dc
->state
= D_PARTIAL
;
1342 spin_unlock_irqrestore(&dc
->lock
, flags
);
1344 atomic_inc(&dcc
->queued_discard
);
1346 list_move_tail(&dc
->list
, wait_list
);
1348 /* sanity check on discard range */
1349 __check_sit_bitmap(sbi
, lstart
, lstart
+ len
);
1351 bio
->bi_private
= dc
;
1352 bio
->bi_end_io
= f2fs_submit_discard_endio
;
1353 bio
->bi_opf
|= flag
;
1356 atomic_inc(&dcc
->issued_discard
);
1358 f2fs_update_iostat(sbi
, NULL
, FS_DISCARD_IO
, len
* F2FS_BLKSIZE
);
1367 dcc
->undiscard_blks
-= len
;
1368 __update_discard_tree_range(sbi
, bdev
, lstart
, start
, len
);
1373 static void __insert_discard_cmd(struct f2fs_sb_info
*sbi
,
1374 struct block_device
*bdev
, block_t lstart
,
1375 block_t start
, block_t len
)
1377 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1378 struct rb_node
**p
= &dcc
->root
.rb_root
.rb_node
;
1379 struct rb_node
*parent
= NULL
;
1380 struct discard_cmd
*dc
;
1381 bool leftmost
= true;
1383 /* look up rb tree to find parent node */
1386 dc
= rb_entry(parent
, struct discard_cmd
, rb_node
);
1388 if (lstart
< dc
->di
.lstart
) {
1390 } else if (lstart
>= dc
->di
.lstart
+ dc
->di
.len
) {
1391 p
= &(*p
)->rb_right
;
1394 /* Let's skip to add, if exists */
1399 dc
= __create_discard_cmd(sbi
, bdev
, lstart
, start
, len
);
1401 rb_link_node(&dc
->rb_node
, parent
, p
);
1402 rb_insert_color_cached(&dc
->rb_node
, &dcc
->root
, leftmost
);
1405 static void __relocate_discard_cmd(struct discard_cmd_control
*dcc
,
1406 struct discard_cmd
*dc
)
1408 list_move_tail(&dc
->list
, &dcc
->pend_list
[plist_idx(dc
->di
.len
)]);
1411 static void __punch_discard_cmd(struct f2fs_sb_info
*sbi
,
1412 struct discard_cmd
*dc
, block_t blkaddr
)
1414 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1415 struct discard_info di
= dc
->di
;
1416 bool modified
= false;
1418 if (dc
->state
== D_DONE
|| dc
->di
.len
== 1) {
1419 __remove_discard_cmd(sbi
, dc
);
1423 dcc
->undiscard_blks
-= di
.len
;
1425 if (blkaddr
> di
.lstart
) {
1426 dc
->di
.len
= blkaddr
- dc
->di
.lstart
;
1427 dcc
->undiscard_blks
+= dc
->di
.len
;
1428 __relocate_discard_cmd(dcc
, dc
);
1432 if (blkaddr
< di
.lstart
+ di
.len
- 1) {
1434 __insert_discard_cmd(sbi
, dc
->bdev
, blkaddr
+ 1,
1435 di
.start
+ blkaddr
+ 1 - di
.lstart
,
1436 di
.lstart
+ di
.len
- 1 - blkaddr
);
1441 dcc
->undiscard_blks
+= dc
->di
.len
;
1442 __relocate_discard_cmd(dcc
, dc
);
1447 static void __update_discard_tree_range(struct f2fs_sb_info
*sbi
,
1448 struct block_device
*bdev
, block_t lstart
,
1449 block_t start
, block_t len
)
1451 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1452 struct discard_cmd
*prev_dc
= NULL
, *next_dc
= NULL
;
1453 struct discard_cmd
*dc
;
1454 struct discard_info di
= {0};
1455 struct rb_node
**insert_p
= NULL
, *insert_parent
= NULL
;
1456 unsigned int max_discard_blocks
=
1457 SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev
));
1458 block_t end
= lstart
+ len
;
1460 dc
= __lookup_discard_cmd_ret(&dcc
->root
, lstart
,
1461 &prev_dc
, &next_dc
, &insert_p
, &insert_parent
);
1467 di
.len
= next_dc
? next_dc
->di
.lstart
- lstart
: len
;
1468 di
.len
= min(di
.len
, len
);
1473 struct rb_node
*node
;
1474 bool merged
= false;
1475 struct discard_cmd
*tdc
= NULL
;
1478 di
.lstart
= prev_dc
->di
.lstart
+ prev_dc
->di
.len
;
1479 if (di
.lstart
< lstart
)
1481 if (di
.lstart
>= end
)
1484 if (!next_dc
|| next_dc
->di
.lstart
> end
)
1485 di
.len
= end
- di
.lstart
;
1487 di
.len
= next_dc
->di
.lstart
- di
.lstart
;
1488 di
.start
= start
+ di
.lstart
- lstart
;
1494 if (prev_dc
&& prev_dc
->state
== D_PREP
&&
1495 prev_dc
->bdev
== bdev
&&
1496 __is_discard_back_mergeable(&di
, &prev_dc
->di
,
1497 max_discard_blocks
)) {
1498 prev_dc
->di
.len
+= di
.len
;
1499 dcc
->undiscard_blks
+= di
.len
;
1500 __relocate_discard_cmd(dcc
, prev_dc
);
1506 if (next_dc
&& next_dc
->state
== D_PREP
&&
1507 next_dc
->bdev
== bdev
&&
1508 __is_discard_front_mergeable(&di
, &next_dc
->di
,
1509 max_discard_blocks
)) {
1510 next_dc
->di
.lstart
= di
.lstart
;
1511 next_dc
->di
.len
+= di
.len
;
1512 next_dc
->di
.start
= di
.start
;
1513 dcc
->undiscard_blks
+= di
.len
;
1514 __relocate_discard_cmd(dcc
, next_dc
);
1516 __remove_discard_cmd(sbi
, tdc
);
1521 __insert_discard_cmd(sbi
, bdev
,
1522 di
.lstart
, di
.start
, di
.len
);
1528 node
= rb_next(&prev_dc
->rb_node
);
1529 next_dc
= rb_entry_safe(node
, struct discard_cmd
, rb_node
);
1533 #ifdef CONFIG_BLK_DEV_ZONED
1534 static void __queue_zone_reset_cmd(struct f2fs_sb_info
*sbi
,
1535 struct block_device
*bdev
, block_t blkstart
, block_t lblkstart
,
1538 trace_f2fs_queue_reset_zone(bdev
, blkstart
);
1540 mutex_lock(&SM_I(sbi
)->dcc_info
->cmd_lock
);
1541 __insert_discard_cmd(sbi
, bdev
, lblkstart
, blkstart
, blklen
);
1542 mutex_unlock(&SM_I(sbi
)->dcc_info
->cmd_lock
);
1546 static void __queue_discard_cmd(struct f2fs_sb_info
*sbi
,
1547 struct block_device
*bdev
, block_t blkstart
, block_t blklen
)
1549 block_t lblkstart
= blkstart
;
1551 if (!f2fs_bdev_support_discard(bdev
))
1554 trace_f2fs_queue_discard(bdev
, blkstart
, blklen
);
1556 if (f2fs_is_multi_device(sbi
)) {
1557 int devi
= f2fs_target_device_index(sbi
, blkstart
);
1559 blkstart
-= FDEV(devi
).start_blk
;
1561 mutex_lock(&SM_I(sbi
)->dcc_info
->cmd_lock
);
1562 __update_discard_tree_range(sbi
, bdev
, lblkstart
, blkstart
, blklen
);
1563 mutex_unlock(&SM_I(sbi
)->dcc_info
->cmd_lock
);
1566 static void __issue_discard_cmd_orderly(struct f2fs_sb_info
*sbi
,
1567 struct discard_policy
*dpolicy
, int *issued
)
1569 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1570 struct discard_cmd
*prev_dc
= NULL
, *next_dc
= NULL
;
1571 struct rb_node
**insert_p
= NULL
, *insert_parent
= NULL
;
1572 struct discard_cmd
*dc
;
1573 struct blk_plug plug
;
1574 bool io_interrupted
= false;
1576 mutex_lock(&dcc
->cmd_lock
);
1577 dc
= __lookup_discard_cmd_ret(&dcc
->root
, dcc
->next_pos
,
1578 &prev_dc
, &next_dc
, &insert_p
, &insert_parent
);
1582 blk_start_plug(&plug
);
1585 struct rb_node
*node
;
1588 if (dc
->state
!= D_PREP
)
1591 if (dpolicy
->io_aware
&& !is_idle(sbi
, DISCARD_TIME
)) {
1592 io_interrupted
= true;
1596 dcc
->next_pos
= dc
->di
.lstart
+ dc
->di
.len
;
1597 err
= __submit_discard_cmd(sbi
, dpolicy
, dc
, issued
);
1599 if (*issued
>= dpolicy
->max_requests
)
1602 node
= rb_next(&dc
->rb_node
);
1604 __remove_discard_cmd(sbi
, dc
);
1605 dc
= rb_entry_safe(node
, struct discard_cmd
, rb_node
);
1608 blk_finish_plug(&plug
);
1613 mutex_unlock(&dcc
->cmd_lock
);
1615 if (!(*issued
) && io_interrupted
)
1618 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info
*sbi
,
1619 struct discard_policy
*dpolicy
);
1621 static int __issue_discard_cmd(struct f2fs_sb_info
*sbi
,
1622 struct discard_policy
*dpolicy
)
1624 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1625 struct list_head
*pend_list
;
1626 struct discard_cmd
*dc
, *tmp
;
1627 struct blk_plug plug
;
1629 bool io_interrupted
= false;
1631 if (dpolicy
->timeout
)
1632 f2fs_update_time(sbi
, UMOUNT_DISCARD_TIMEOUT
);
1636 for (i
= MAX_PLIST_NUM
- 1; i
>= 0; i
--) {
1637 if (dpolicy
->timeout
&&
1638 f2fs_time_over(sbi
, UMOUNT_DISCARD_TIMEOUT
))
1641 if (i
+ 1 < dpolicy
->granularity
)
1644 if (i
+ 1 < dcc
->max_ordered_discard
&& dpolicy
->ordered
) {
1645 __issue_discard_cmd_orderly(sbi
, dpolicy
, &issued
);
1649 pend_list
= &dcc
->pend_list
[i
];
1651 mutex_lock(&dcc
->cmd_lock
);
1652 if (list_empty(pend_list
))
1654 if (unlikely(dcc
->rbtree_check
))
1655 f2fs_bug_on(sbi
, !f2fs_check_discard_tree(sbi
));
1656 blk_start_plug(&plug
);
1657 list_for_each_entry_safe(dc
, tmp
, pend_list
, list
) {
1658 f2fs_bug_on(sbi
, dc
->state
!= D_PREP
);
1660 if (dpolicy
->timeout
&&
1661 f2fs_time_over(sbi
, UMOUNT_DISCARD_TIMEOUT
))
1664 if (dpolicy
->io_aware
&& i
< dpolicy
->io_aware_gran
&&
1665 !is_idle(sbi
, DISCARD_TIME
)) {
1666 io_interrupted
= true;
1670 __submit_discard_cmd(sbi
, dpolicy
, dc
, &issued
);
1672 if (issued
>= dpolicy
->max_requests
)
1675 blk_finish_plug(&plug
);
1677 mutex_unlock(&dcc
->cmd_lock
);
1679 if (issued
>= dpolicy
->max_requests
|| io_interrupted
)
1683 if (dpolicy
->type
== DPOLICY_UMOUNT
&& issued
) {
1684 __wait_all_discard_cmd(sbi
, dpolicy
);
1688 if (!issued
&& io_interrupted
)
1694 static bool __drop_discard_cmd(struct f2fs_sb_info
*sbi
)
1696 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1697 struct list_head
*pend_list
;
1698 struct discard_cmd
*dc
, *tmp
;
1700 bool dropped
= false;
1702 mutex_lock(&dcc
->cmd_lock
);
1703 for (i
= MAX_PLIST_NUM
- 1; i
>= 0; i
--) {
1704 pend_list
= &dcc
->pend_list
[i
];
1705 list_for_each_entry_safe(dc
, tmp
, pend_list
, list
) {
1706 f2fs_bug_on(sbi
, dc
->state
!= D_PREP
);
1707 __remove_discard_cmd(sbi
, dc
);
1711 mutex_unlock(&dcc
->cmd_lock
);
1716 void f2fs_drop_discard_cmd(struct f2fs_sb_info
*sbi
)
1718 __drop_discard_cmd(sbi
);
1721 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info
*sbi
,
1722 struct discard_cmd
*dc
)
1724 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1725 unsigned int len
= 0;
1727 wait_for_completion_io(&dc
->wait
);
1728 mutex_lock(&dcc
->cmd_lock
);
1729 f2fs_bug_on(sbi
, dc
->state
!= D_DONE
);
1734 __remove_discard_cmd(sbi
, dc
);
1736 mutex_unlock(&dcc
->cmd_lock
);
1741 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info
*sbi
,
1742 struct discard_policy
*dpolicy
,
1743 block_t start
, block_t end
)
1745 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1746 struct list_head
*wait_list
= (dpolicy
->type
== DPOLICY_FSTRIM
) ?
1747 &(dcc
->fstrim_list
) : &(dcc
->wait_list
);
1748 struct discard_cmd
*dc
= NULL
, *iter
, *tmp
;
1749 unsigned int trimmed
= 0;
1754 mutex_lock(&dcc
->cmd_lock
);
1755 list_for_each_entry_safe(iter
, tmp
, wait_list
, list
) {
1756 if (iter
->di
.lstart
+ iter
->di
.len
<= start
||
1757 end
<= iter
->di
.lstart
)
1759 if (iter
->di
.len
< dpolicy
->granularity
)
1761 if (iter
->state
== D_DONE
&& !iter
->ref
) {
1762 wait_for_completion_io(&iter
->wait
);
1764 trimmed
+= iter
->di
.len
;
1765 __remove_discard_cmd(sbi
, iter
);
1772 mutex_unlock(&dcc
->cmd_lock
);
1775 trimmed
+= __wait_one_discard_bio(sbi
, dc
);
1782 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info
*sbi
,
1783 struct discard_policy
*dpolicy
)
1785 struct discard_policy dp
;
1786 unsigned int discard_blks
;
1789 return __wait_discard_cmd_range(sbi
, dpolicy
, 0, UINT_MAX
);
1792 __init_discard_policy(sbi
, &dp
, DPOLICY_FSTRIM
, MIN_DISCARD_GRANULARITY
);
1793 discard_blks
= __wait_discard_cmd_range(sbi
, &dp
, 0, UINT_MAX
);
1794 __init_discard_policy(sbi
, &dp
, DPOLICY_UMOUNT
, MIN_DISCARD_GRANULARITY
);
1795 discard_blks
+= __wait_discard_cmd_range(sbi
, &dp
, 0, UINT_MAX
);
1797 return discard_blks
;
1800 /* This should be covered by global mutex, &sit_i->sentry_lock */
1801 static void f2fs_wait_discard_bio(struct f2fs_sb_info
*sbi
, block_t blkaddr
)
1803 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1804 struct discard_cmd
*dc
;
1805 bool need_wait
= false;
1807 mutex_lock(&dcc
->cmd_lock
);
1808 dc
= __lookup_discard_cmd(sbi
, blkaddr
);
1809 #ifdef CONFIG_BLK_DEV_ZONED
1810 if (dc
&& f2fs_sb_has_blkzoned(sbi
) && bdev_is_zoned(dc
->bdev
)) {
1811 int devi
= f2fs_bdev_index(sbi
, dc
->bdev
);
1814 mutex_unlock(&dcc
->cmd_lock
);
1818 if (f2fs_blkz_is_seq(sbi
, devi
, dc
->di
.start
)) {
1819 /* force submit zone reset */
1820 if (dc
->state
== D_PREP
)
1821 __submit_zone_reset_cmd(sbi
, dc
, REQ_SYNC
,
1822 &dcc
->wait_list
, NULL
);
1824 mutex_unlock(&dcc
->cmd_lock
);
1825 /* wait zone reset */
1826 __wait_one_discard_bio(sbi
, dc
);
1832 if (dc
->state
== D_PREP
) {
1833 __punch_discard_cmd(sbi
, dc
, blkaddr
);
1839 mutex_unlock(&dcc
->cmd_lock
);
1842 __wait_one_discard_bio(sbi
, dc
);
1845 void f2fs_stop_discard_thread(struct f2fs_sb_info
*sbi
)
1847 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1849 if (dcc
&& dcc
->f2fs_issue_discard
) {
1850 struct task_struct
*discard_thread
= dcc
->f2fs_issue_discard
;
1852 dcc
->f2fs_issue_discard
= NULL
;
1853 kthread_stop(discard_thread
);
1858 * f2fs_issue_discard_timeout() - Issue all discard cmd within UMOUNT_DISCARD_TIMEOUT
1859 * @sbi: the f2fs_sb_info data for discard cmd to issue
1861 * When UMOUNT_DISCARD_TIMEOUT is exceeded, all remaining discard commands will be dropped
1863 * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false.
1865 bool f2fs_issue_discard_timeout(struct f2fs_sb_info
*sbi
)
1867 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1868 struct discard_policy dpolicy
;
1871 if (!atomic_read(&dcc
->discard_cmd_cnt
))
1874 __init_discard_policy(sbi
, &dpolicy
, DPOLICY_UMOUNT
,
1875 dcc
->discard_granularity
);
1876 __issue_discard_cmd(sbi
, &dpolicy
);
1877 dropped
= __drop_discard_cmd(sbi
);
1879 /* just to make sure there is no pending discard commands */
1880 __wait_all_discard_cmd(sbi
, NULL
);
1882 f2fs_bug_on(sbi
, atomic_read(&dcc
->discard_cmd_cnt
));
1886 static int issue_discard_thread(void *data
)
1888 struct f2fs_sb_info
*sbi
= data
;
1889 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
1890 wait_queue_head_t
*q
= &dcc
->discard_wait_queue
;
1891 struct discard_policy dpolicy
;
1892 unsigned int wait_ms
= dcc
->min_discard_issue_time
;
1898 wait_event_freezable_timeout(*q
,
1899 kthread_should_stop() || dcc
->discard_wake
,
1900 msecs_to_jiffies(wait_ms
));
1902 if (sbi
->gc_mode
== GC_URGENT_HIGH
||
1903 !f2fs_available_free_memory(sbi
, DISCARD_CACHE
))
1904 __init_discard_policy(sbi
, &dpolicy
, DPOLICY_FORCE
,
1905 MIN_DISCARD_GRANULARITY
);
1907 __init_discard_policy(sbi
, &dpolicy
, DPOLICY_BG
,
1908 dcc
->discard_granularity
);
1910 if (dcc
->discard_wake
)
1911 dcc
->discard_wake
= false;
1913 /* clean up pending candidates before going to sleep */
1914 if (atomic_read(&dcc
->queued_discard
))
1915 __wait_all_discard_cmd(sbi
, NULL
);
1917 if (f2fs_readonly(sbi
->sb
))
1919 if (kthread_should_stop())
1921 if (is_sbi_flag_set(sbi
, SBI_NEED_FSCK
) ||
1922 !atomic_read(&dcc
->discard_cmd_cnt
)) {
1923 wait_ms
= dpolicy
.max_interval
;
1927 sb_start_intwrite(sbi
->sb
);
1929 issued
= __issue_discard_cmd(sbi
, &dpolicy
);
1931 __wait_all_discard_cmd(sbi
, &dpolicy
);
1932 wait_ms
= dpolicy
.min_interval
;
1933 } else if (issued
== -1) {
1934 wait_ms
= f2fs_time_to_wait(sbi
, DISCARD_TIME
);
1936 wait_ms
= dpolicy
.mid_interval
;
1938 wait_ms
= dpolicy
.max_interval
;
1940 if (!atomic_read(&dcc
->discard_cmd_cnt
))
1941 wait_ms
= dpolicy
.max_interval
;
1943 sb_end_intwrite(sbi
->sb
);
1945 } while (!kthread_should_stop());
1949 #ifdef CONFIG_BLK_DEV_ZONED
1950 static int __f2fs_issue_discard_zone(struct f2fs_sb_info
*sbi
,
1951 struct block_device
*bdev
, block_t blkstart
, block_t blklen
)
1953 sector_t sector
, nr_sects
;
1954 block_t lblkstart
= blkstart
;
1958 if (f2fs_is_multi_device(sbi
)) {
1959 devi
= f2fs_target_device_index(sbi
, blkstart
);
1960 if (blkstart
< FDEV(devi
).start_blk
||
1961 blkstart
> FDEV(devi
).end_blk
) {
1962 f2fs_err(sbi
, "Invalid block %x", blkstart
);
1965 blkstart
-= FDEV(devi
).start_blk
;
1968 /* For sequential zones, reset the zone write pointer */
1969 if (f2fs_blkz_is_seq(sbi
, devi
, blkstart
)) {
1970 sector
= SECTOR_FROM_BLOCK(blkstart
);
1971 nr_sects
= SECTOR_FROM_BLOCK(blklen
);
1972 div64_u64_rem(sector
, bdev_zone_sectors(bdev
), &remainder
);
1974 if (remainder
|| nr_sects
!= bdev_zone_sectors(bdev
)) {
1975 f2fs_err(sbi
, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
1976 devi
, sbi
->s_ndevs
? FDEV(devi
).path
: "",
1981 if (unlikely(is_sbi_flag_set(sbi
, SBI_POR_DOING
))) {
1982 unsigned int nofs_flags
;
1985 trace_f2fs_issue_reset_zone(bdev
, blkstart
);
1986 nofs_flags
= memalloc_nofs_save();
1987 ret
= blkdev_zone_mgmt(bdev
, REQ_OP_ZONE_RESET
,
1989 memalloc_nofs_restore(nofs_flags
);
1993 __queue_zone_reset_cmd(sbi
, bdev
, blkstart
, lblkstart
, blklen
);
1997 /* For conventional zones, use regular discard if supported */
1998 __queue_discard_cmd(sbi
, bdev
, lblkstart
, blklen
);
2003 static int __issue_discard_async(struct f2fs_sb_info
*sbi
,
2004 struct block_device
*bdev
, block_t blkstart
, block_t blklen
)
2006 #ifdef CONFIG_BLK_DEV_ZONED
2007 if (f2fs_sb_has_blkzoned(sbi
) && bdev_is_zoned(bdev
))
2008 return __f2fs_issue_discard_zone(sbi
, bdev
, blkstart
, blklen
);
2010 __queue_discard_cmd(sbi
, bdev
, blkstart
, blklen
);
2014 static int f2fs_issue_discard(struct f2fs_sb_info
*sbi
,
2015 block_t blkstart
, block_t blklen
)
2017 sector_t start
= blkstart
, len
= 0;
2018 struct block_device
*bdev
;
2019 struct seg_entry
*se
;
2020 unsigned int offset
;
2024 bdev
= f2fs_target_device(sbi
, blkstart
, NULL
);
2026 for (i
= blkstart
; i
< blkstart
+ blklen
; i
++, len
++) {
2028 struct block_device
*bdev2
=
2029 f2fs_target_device(sbi
, i
, NULL
);
2031 if (bdev2
!= bdev
) {
2032 err
= __issue_discard_async(sbi
, bdev
,
2042 se
= get_seg_entry(sbi
, GET_SEGNO(sbi
, i
));
2043 offset
= GET_BLKOFF_FROM_SEG0(sbi
, i
);
2045 if (f2fs_block_unit_discard(sbi
) &&
2046 !f2fs_test_and_set_bit(offset
, se
->discard_map
))
2047 sbi
->discard_blks
--;
2051 err
= __issue_discard_async(sbi
, bdev
, start
, len
);
2055 static bool add_discard_addrs(struct f2fs_sb_info
*sbi
, struct cp_control
*cpc
,
2058 int entries
= SIT_VBLOCK_MAP_SIZE
/ sizeof(unsigned long);
2059 struct seg_entry
*se
= get_seg_entry(sbi
, cpc
->trim_start
);
2060 unsigned long *cur_map
= (unsigned long *)se
->cur_valid_map
;
2061 unsigned long *ckpt_map
= (unsigned long *)se
->ckpt_valid_map
;
2062 unsigned long *discard_map
= (unsigned long *)se
->discard_map
;
2063 unsigned long *dmap
= SIT_I(sbi
)->tmp_map
;
2064 unsigned int start
= 0, end
= -1;
2065 bool force
= (cpc
->reason
& CP_DISCARD
);
2066 struct discard_entry
*de
= NULL
;
2067 struct list_head
*head
= &SM_I(sbi
)->dcc_info
->entry_list
;
2070 if (se
->valid_blocks
== BLKS_PER_SEG(sbi
) ||
2071 !f2fs_hw_support_discard(sbi
) ||
2072 !f2fs_block_unit_discard(sbi
))
2076 if (!f2fs_realtime_discard_enable(sbi
) || !se
->valid_blocks
||
2077 SM_I(sbi
)->dcc_info
->nr_discards
>=
2078 SM_I(sbi
)->dcc_info
->max_discards
)
2082 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
2083 for (i
= 0; i
< entries
; i
++)
2084 dmap
[i
] = force
? ~ckpt_map
[i
] & ~discard_map
[i
] :
2085 (cur_map
[i
] ^ ckpt_map
[i
]) & ckpt_map
[i
];
2087 while (force
|| SM_I(sbi
)->dcc_info
->nr_discards
<=
2088 SM_I(sbi
)->dcc_info
->max_discards
) {
2089 start
= __find_rev_next_bit(dmap
, BLKS_PER_SEG(sbi
), end
+ 1);
2090 if (start
>= BLKS_PER_SEG(sbi
))
2093 end
= __find_rev_next_zero_bit(dmap
,
2094 BLKS_PER_SEG(sbi
), start
+ 1);
2095 if (force
&& start
&& end
!= BLKS_PER_SEG(sbi
) &&
2096 (end
- start
) < cpc
->trim_minlen
)
2103 de
= f2fs_kmem_cache_alloc(discard_entry_slab
,
2104 GFP_F2FS_ZERO
, true, NULL
);
2105 de
->start_blkaddr
= START_BLOCK(sbi
, cpc
->trim_start
);
2106 list_add_tail(&de
->list
, head
);
2109 for (i
= start
; i
< end
; i
++)
2110 __set_bit_le(i
, (void *)de
->discard_map
);
2112 SM_I(sbi
)->dcc_info
->nr_discards
+= end
- start
;
2117 static void release_discard_addr(struct discard_entry
*entry
)
2119 list_del(&entry
->list
);
2120 kmem_cache_free(discard_entry_slab
, entry
);
2123 void f2fs_release_discard_addrs(struct f2fs_sb_info
*sbi
)
2125 struct list_head
*head
= &(SM_I(sbi
)->dcc_info
->entry_list
);
2126 struct discard_entry
*entry
, *this;
2129 list_for_each_entry_safe(entry
, this, head
, list
)
2130 release_discard_addr(entry
);
2134 * Should call f2fs_clear_prefree_segments after checkpoint is done.
2136 static void set_prefree_as_free_segments(struct f2fs_sb_info
*sbi
)
2138 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
2141 mutex_lock(&dirty_i
->seglist_lock
);
2142 for_each_set_bit(segno
, dirty_i
->dirty_segmap
[PRE
], MAIN_SEGS(sbi
))
2143 __set_test_and_free(sbi
, segno
, false);
2144 mutex_unlock(&dirty_i
->seglist_lock
);
2147 void f2fs_clear_prefree_segments(struct f2fs_sb_info
*sbi
,
2148 struct cp_control
*cpc
)
2150 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
2151 struct list_head
*head
= &dcc
->entry_list
;
2152 struct discard_entry
*entry
, *this;
2153 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
2154 unsigned long *prefree_map
= dirty_i
->dirty_segmap
[PRE
];
2155 unsigned int start
= 0, end
= -1;
2156 unsigned int secno
, start_segno
;
2157 bool force
= (cpc
->reason
& CP_DISCARD
);
2158 bool section_alignment
= F2FS_OPTION(sbi
).discard_unit
==
2159 DISCARD_UNIT_SECTION
;
2161 if (f2fs_lfs_mode(sbi
) && __is_large_section(sbi
))
2162 section_alignment
= true;
2164 mutex_lock(&dirty_i
->seglist_lock
);
2169 if (section_alignment
&& end
!= -1)
2171 start
= find_next_bit(prefree_map
, MAIN_SEGS(sbi
), end
+ 1);
2172 if (start
>= MAIN_SEGS(sbi
))
2174 end
= find_next_zero_bit(prefree_map
, MAIN_SEGS(sbi
),
2177 if (section_alignment
) {
2178 start
= rounddown(start
, SEGS_PER_SEC(sbi
));
2179 end
= roundup(end
, SEGS_PER_SEC(sbi
));
2182 for (i
= start
; i
< end
; i
++) {
2183 if (test_and_clear_bit(i
, prefree_map
))
2184 dirty_i
->nr_dirty
[PRE
]--;
2187 if (!f2fs_realtime_discard_enable(sbi
))
2190 if (force
&& start
>= cpc
->trim_start
&&
2191 (end
- 1) <= cpc
->trim_end
)
2194 /* Should cover 2MB zoned device for zone-based reset */
2195 if (!f2fs_sb_has_blkzoned(sbi
) &&
2196 (!f2fs_lfs_mode(sbi
) || !__is_large_section(sbi
))) {
2197 f2fs_issue_discard(sbi
, START_BLOCK(sbi
, start
),
2198 SEGS_TO_BLKS(sbi
, end
- start
));
2202 secno
= GET_SEC_FROM_SEG(sbi
, start
);
2203 start_segno
= GET_SEG_FROM_SEC(sbi
, secno
);
2204 if (!IS_CURSEC(sbi
, secno
) &&
2205 !get_valid_blocks(sbi
, start
, true))
2206 f2fs_issue_discard(sbi
, START_BLOCK(sbi
, start_segno
),
2209 start
= start_segno
+ SEGS_PER_SEC(sbi
);
2215 mutex_unlock(&dirty_i
->seglist_lock
);
2217 if (!f2fs_block_unit_discard(sbi
))
2220 /* send small discards */
2221 list_for_each_entry_safe(entry
, this, head
, list
) {
2222 unsigned int cur_pos
= 0, next_pos
, len
, total_len
= 0;
2223 bool is_valid
= test_bit_le(0, entry
->discard_map
);
2227 next_pos
= find_next_zero_bit_le(entry
->discard_map
,
2228 BLKS_PER_SEG(sbi
), cur_pos
);
2229 len
= next_pos
- cur_pos
;
2231 if (f2fs_sb_has_blkzoned(sbi
) ||
2232 (force
&& len
< cpc
->trim_minlen
))
2235 f2fs_issue_discard(sbi
, entry
->start_blkaddr
+ cur_pos
,
2239 next_pos
= find_next_bit_le(entry
->discard_map
,
2240 BLKS_PER_SEG(sbi
), cur_pos
);
2244 is_valid
= !is_valid
;
2246 if (cur_pos
< BLKS_PER_SEG(sbi
))
2249 release_discard_addr(entry
);
2250 dcc
->nr_discards
-= total_len
;
2254 wake_up_discard_thread(sbi
, false);
2257 int f2fs_start_discard_thread(struct f2fs_sb_info
*sbi
)
2259 dev_t dev
= sbi
->sb
->s_bdev
->bd_dev
;
2260 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
2263 if (f2fs_sb_has_readonly(sbi
)) {
2265 "Skip to start discard thread for readonly image");
2269 if (!f2fs_realtime_discard_enable(sbi
))
2272 dcc
->f2fs_issue_discard
= kthread_run(issue_discard_thread
, sbi
,
2273 "f2fs_discard-%u:%u", MAJOR(dev
), MINOR(dev
));
2274 if (IS_ERR(dcc
->f2fs_issue_discard
)) {
2275 err
= PTR_ERR(dcc
->f2fs_issue_discard
);
2276 dcc
->f2fs_issue_discard
= NULL
;
2282 static int create_discard_cmd_control(struct f2fs_sb_info
*sbi
)
2284 struct discard_cmd_control
*dcc
;
2287 if (SM_I(sbi
)->dcc_info
) {
2288 dcc
= SM_I(sbi
)->dcc_info
;
2292 dcc
= f2fs_kzalloc(sbi
, sizeof(struct discard_cmd_control
), GFP_KERNEL
);
2296 dcc
->discard_io_aware_gran
= MAX_PLIST_NUM
;
2297 dcc
->discard_granularity
= DEFAULT_DISCARD_GRANULARITY
;
2298 dcc
->max_ordered_discard
= DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY
;
2299 dcc
->discard_io_aware
= DPOLICY_IO_AWARE_ENABLE
;
2300 if (F2FS_OPTION(sbi
).discard_unit
== DISCARD_UNIT_SEGMENT
)
2301 dcc
->discard_granularity
= BLKS_PER_SEG(sbi
);
2302 else if (F2FS_OPTION(sbi
).discard_unit
== DISCARD_UNIT_SECTION
)
2303 dcc
->discard_granularity
= BLKS_PER_SEC(sbi
);
2305 INIT_LIST_HEAD(&dcc
->entry_list
);
2306 for (i
= 0; i
< MAX_PLIST_NUM
; i
++)
2307 INIT_LIST_HEAD(&dcc
->pend_list
[i
]);
2308 INIT_LIST_HEAD(&dcc
->wait_list
);
2309 INIT_LIST_HEAD(&dcc
->fstrim_list
);
2310 mutex_init(&dcc
->cmd_lock
);
2311 atomic_set(&dcc
->issued_discard
, 0);
2312 atomic_set(&dcc
->queued_discard
, 0);
2313 atomic_set(&dcc
->discard_cmd_cnt
, 0);
2314 dcc
->nr_discards
= 0;
2315 dcc
->max_discards
= SEGS_TO_BLKS(sbi
, MAIN_SEGS(sbi
));
2316 dcc
->max_discard_request
= DEF_MAX_DISCARD_REQUEST
;
2317 dcc
->min_discard_issue_time
= DEF_MIN_DISCARD_ISSUE_TIME
;
2318 dcc
->mid_discard_issue_time
= DEF_MID_DISCARD_ISSUE_TIME
;
2319 dcc
->max_discard_issue_time
= DEF_MAX_DISCARD_ISSUE_TIME
;
2320 dcc
->discard_urgent_util
= DEF_DISCARD_URGENT_UTIL
;
2321 dcc
->undiscard_blks
= 0;
2323 dcc
->root
= RB_ROOT_CACHED
;
2324 dcc
->rbtree_check
= false;
2326 init_waitqueue_head(&dcc
->discard_wait_queue
);
2327 SM_I(sbi
)->dcc_info
= dcc
;
2329 err
= f2fs_start_discard_thread(sbi
);
2332 SM_I(sbi
)->dcc_info
= NULL
;
2338 static void destroy_discard_cmd_control(struct f2fs_sb_info
*sbi
)
2340 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
2345 f2fs_stop_discard_thread(sbi
);
2348 * Recovery can cache discard commands, so in error path of
2349 * fill_super(), it needs to give a chance to handle them.
2351 f2fs_issue_discard_timeout(sbi
);
2354 SM_I(sbi
)->dcc_info
= NULL
;
2357 static bool __mark_sit_entry_dirty(struct f2fs_sb_info
*sbi
, unsigned int segno
)
2359 struct sit_info
*sit_i
= SIT_I(sbi
);
2361 if (!__test_and_set_bit(segno
, sit_i
->dirty_sentries_bitmap
)) {
2362 sit_i
->dirty_sentries
++;
2369 static void __set_sit_entry_type(struct f2fs_sb_info
*sbi
, int type
,
2370 unsigned int segno
, int modified
)
2372 struct seg_entry
*se
= get_seg_entry(sbi
, segno
);
2376 __mark_sit_entry_dirty(sbi
, segno
);
2379 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info
*sbi
,
2382 unsigned int segno
= GET_SEGNO(sbi
, blkaddr
);
2384 if (segno
== NULL_SEGNO
)
2386 return get_seg_entry(sbi
, segno
)->mtime
;
2389 static void update_segment_mtime(struct f2fs_sb_info
*sbi
, block_t blkaddr
,
2390 unsigned long long old_mtime
)
2392 struct seg_entry
*se
;
2393 unsigned int segno
= GET_SEGNO(sbi
, blkaddr
);
2394 unsigned long long ctime
= get_mtime(sbi
, false);
2395 unsigned long long mtime
= old_mtime
? old_mtime
: ctime
;
2397 if (segno
== NULL_SEGNO
)
2400 se
= get_seg_entry(sbi
, segno
);
2405 se
->mtime
= div_u64(se
->mtime
* se
->valid_blocks
+ mtime
,
2406 se
->valid_blocks
+ 1);
2408 if (ctime
> SIT_I(sbi
)->max_mtime
)
2409 SIT_I(sbi
)->max_mtime
= ctime
;
2412 static void update_sit_entry(struct f2fs_sb_info
*sbi
, block_t blkaddr
, int del
)
2414 struct seg_entry
*se
;
2415 unsigned int segno
, offset
;
2416 long int new_vblocks
;
2418 #ifdef CONFIG_F2FS_CHECK_FS
2422 segno
= GET_SEGNO(sbi
, blkaddr
);
2423 if (segno
== NULL_SEGNO
)
2426 se
= get_seg_entry(sbi
, segno
);
2427 new_vblocks
= se
->valid_blocks
+ del
;
2428 offset
= GET_BLKOFF_FROM_SEG0(sbi
, blkaddr
);
2430 f2fs_bug_on(sbi
, (new_vblocks
< 0 ||
2431 (new_vblocks
> f2fs_usable_blks_in_seg(sbi
, segno
))));
2433 se
->valid_blocks
= new_vblocks
;
2435 /* Update valid block bitmap */
2437 exist
= f2fs_test_and_set_bit(offset
, se
->cur_valid_map
);
2438 #ifdef CONFIG_F2FS_CHECK_FS
2439 mir_exist
= f2fs_test_and_set_bit(offset
,
2440 se
->cur_valid_map_mir
);
2441 if (unlikely(exist
!= mir_exist
)) {
2442 f2fs_err(sbi
, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
2444 f2fs_bug_on(sbi
, 1);
2447 if (unlikely(exist
)) {
2448 f2fs_err(sbi
, "Bitmap was wrongly set, blk:%u",
2450 f2fs_bug_on(sbi
, 1);
2455 if (f2fs_block_unit_discard(sbi
) &&
2456 !f2fs_test_and_set_bit(offset
, se
->discard_map
))
2457 sbi
->discard_blks
--;
2460 * SSR should never reuse block which is checkpointed
2461 * or newly invalidated.
2463 if (!is_sbi_flag_set(sbi
, SBI_CP_DISABLED
)) {
2464 if (!f2fs_test_and_set_bit(offset
, se
->ckpt_valid_map
))
2465 se
->ckpt_valid_blocks
++;
2468 exist
= f2fs_test_and_clear_bit(offset
, se
->cur_valid_map
);
2469 #ifdef CONFIG_F2FS_CHECK_FS
2470 mir_exist
= f2fs_test_and_clear_bit(offset
,
2471 se
->cur_valid_map_mir
);
2472 if (unlikely(exist
!= mir_exist
)) {
2473 f2fs_err(sbi
, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
2475 f2fs_bug_on(sbi
, 1);
2478 if (unlikely(!exist
)) {
2479 f2fs_err(sbi
, "Bitmap was wrongly cleared, blk:%u",
2481 f2fs_bug_on(sbi
, 1);
2484 } else if (unlikely(is_sbi_flag_set(sbi
, SBI_CP_DISABLED
))) {
2486 * If checkpoints are off, we must not reuse data that
2487 * was used in the previous checkpoint. If it was used
2488 * before, we must track that to know how much space we
2491 if (f2fs_test_bit(offset
, se
->ckpt_valid_map
)) {
2492 spin_lock(&sbi
->stat_lock
);
2493 sbi
->unusable_block_count
++;
2494 spin_unlock(&sbi
->stat_lock
);
2498 if (f2fs_block_unit_discard(sbi
) &&
2499 f2fs_test_and_clear_bit(offset
, se
->discard_map
))
2500 sbi
->discard_blks
++;
2502 if (!f2fs_test_bit(offset
, se
->ckpt_valid_map
))
2503 se
->ckpt_valid_blocks
+= del
;
2505 __mark_sit_entry_dirty(sbi
, segno
);
2507 /* update total number of valid blocks to be written in ckpt area */
2508 SIT_I(sbi
)->written_valid_blocks
+= del
;
2510 if (__is_large_section(sbi
))
2511 get_sec_entry(sbi
, segno
)->valid_blocks
+= del
;
2514 void f2fs_invalidate_blocks(struct f2fs_sb_info
*sbi
, block_t addr
)
2516 unsigned int segno
= GET_SEGNO(sbi
, addr
);
2517 struct sit_info
*sit_i
= SIT_I(sbi
);
2519 f2fs_bug_on(sbi
, addr
== NULL_ADDR
);
2520 if (addr
== NEW_ADDR
|| addr
== COMPRESS_ADDR
)
2523 f2fs_invalidate_internal_cache(sbi
, addr
);
2525 /* add it into sit main buffer */
2526 down_write(&sit_i
->sentry_lock
);
2528 update_segment_mtime(sbi
, addr
, 0);
2529 update_sit_entry(sbi
, addr
, -1);
2531 /* add it into dirty seglist */
2532 locate_dirty_segment(sbi
, segno
);
2534 up_write(&sit_i
->sentry_lock
);
2537 bool f2fs_is_checkpointed_data(struct f2fs_sb_info
*sbi
, block_t blkaddr
)
2539 struct sit_info
*sit_i
= SIT_I(sbi
);
2540 unsigned int segno
, offset
;
2541 struct seg_entry
*se
;
2544 if (!__is_valid_data_blkaddr(blkaddr
))
2547 down_read(&sit_i
->sentry_lock
);
2549 segno
= GET_SEGNO(sbi
, blkaddr
);
2550 se
= get_seg_entry(sbi
, segno
);
2551 offset
= GET_BLKOFF_FROM_SEG0(sbi
, blkaddr
);
2553 if (f2fs_test_bit(offset
, se
->ckpt_valid_map
))
2556 up_read(&sit_i
->sentry_lock
);
2561 static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info
*sbi
, int type
)
2563 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
2565 if (sbi
->ckpt
->alloc_type
[type
] == SSR
)
2566 return BLKS_PER_SEG(sbi
);
2567 return curseg
->next_blkoff
;
2571 * Calculate the number of current summary pages for writing
2573 int f2fs_npages_for_summary_flush(struct f2fs_sb_info
*sbi
, bool for_ra
)
2575 int valid_sum_count
= 0;
2578 for (i
= CURSEG_HOT_DATA
; i
<= CURSEG_COLD_DATA
; i
++) {
2579 if (sbi
->ckpt
->alloc_type
[i
] != SSR
&& for_ra
)
2581 le16_to_cpu(F2FS_CKPT(sbi
)->cur_data_blkoff
[i
]);
2583 valid_sum_count
+= f2fs_curseg_valid_blocks(sbi
, i
);
2586 sum_in_page
= (PAGE_SIZE
- 2 * SUM_JOURNAL_SIZE
-
2587 SUM_FOOTER_SIZE
) / SUMMARY_SIZE
;
2588 if (valid_sum_count
<= sum_in_page
)
2590 else if ((valid_sum_count
- sum_in_page
) <=
2591 (PAGE_SIZE
- SUM_FOOTER_SIZE
) / SUMMARY_SIZE
)
2597 * Caller should put this summary page
2599 struct page
*f2fs_get_sum_page(struct f2fs_sb_info
*sbi
, unsigned int segno
)
2601 if (unlikely(f2fs_cp_error(sbi
)))
2602 return ERR_PTR(-EIO
);
2603 return f2fs_get_meta_page_retry(sbi
, GET_SUM_BLOCK(sbi
, segno
));
2606 void f2fs_update_meta_page(struct f2fs_sb_info
*sbi
,
2607 void *src
, block_t blk_addr
)
2609 struct page
*page
= f2fs_grab_meta_page(sbi
, blk_addr
);
2611 memcpy(page_address(page
), src
, PAGE_SIZE
);
2612 set_page_dirty(page
);
2613 f2fs_put_page(page
, 1);
2616 static void write_sum_page(struct f2fs_sb_info
*sbi
,
2617 struct f2fs_summary_block
*sum_blk
, block_t blk_addr
)
2619 f2fs_update_meta_page(sbi
, (void *)sum_blk
, blk_addr
);
2622 static void write_current_sum_page(struct f2fs_sb_info
*sbi
,
2623 int type
, block_t blk_addr
)
2625 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
2626 struct page
*page
= f2fs_grab_meta_page(sbi
, blk_addr
);
2627 struct f2fs_summary_block
*src
= curseg
->sum_blk
;
2628 struct f2fs_summary_block
*dst
;
2630 dst
= (struct f2fs_summary_block
*)page_address(page
);
2631 memset(dst
, 0, PAGE_SIZE
);
2633 mutex_lock(&curseg
->curseg_mutex
);
2635 down_read(&curseg
->journal_rwsem
);
2636 memcpy(&dst
->journal
, curseg
->journal
, SUM_JOURNAL_SIZE
);
2637 up_read(&curseg
->journal_rwsem
);
2639 memcpy(dst
->entries
, src
->entries
, SUM_ENTRY_SIZE
);
2640 memcpy(&dst
->footer
, &src
->footer
, SUM_FOOTER_SIZE
);
2642 mutex_unlock(&curseg
->curseg_mutex
);
2644 set_page_dirty(page
);
2645 f2fs_put_page(page
, 1);
2648 static int is_next_segment_free(struct f2fs_sb_info
*sbi
,
2649 struct curseg_info
*curseg
)
2651 unsigned int segno
= curseg
->segno
+ 1;
2652 struct free_segmap_info
*free_i
= FREE_I(sbi
);
2654 if (segno
< MAIN_SEGS(sbi
) && segno
% SEGS_PER_SEC(sbi
))
2655 return !test_bit(segno
, free_i
->free_segmap
);
2660 * Find a new segment from the free segments bitmap to right order
2661 * This function should be returned with success, otherwise BUG
2663 static int get_new_segment(struct f2fs_sb_info
*sbi
,
2664 unsigned int *newseg
, bool new_sec
, bool pinning
)
2666 struct free_segmap_info
*free_i
= FREE_I(sbi
);
2667 unsigned int segno
, secno
, zoneno
;
2668 unsigned int total_zones
= MAIN_SECS(sbi
) / sbi
->secs_per_zone
;
2669 unsigned int hint
= GET_SEC_FROM_SEG(sbi
, *newseg
);
2670 unsigned int old_zoneno
= GET_ZONE_FROM_SEG(sbi
, *newseg
);
2675 spin_lock(&free_i
->segmap_lock
);
2677 if (time_to_inject(sbi
, FAULT_NO_SEGMENT
)) {
2682 if (!new_sec
&& ((*newseg
+ 1) % SEGS_PER_SEC(sbi
))) {
2683 segno
= find_next_zero_bit(free_i
->free_segmap
,
2684 GET_SEG_FROM_SEC(sbi
, hint
+ 1), *newseg
+ 1);
2685 if (segno
< GET_SEG_FROM_SEC(sbi
, hint
+ 1))
2690 * If we format f2fs on zoned storage, let's try to get pinned sections
2691 * from beginning of the storage, which should be a conventional one.
2693 if (f2fs_sb_has_blkzoned(sbi
)) {
2694 segno
= pinning
? 0 : max(first_zoned_segno(sbi
), *newseg
);
2695 hint
= GET_SEC_FROM_SEG(sbi
, segno
);
2699 secno
= find_next_zero_bit(free_i
->free_secmap
, MAIN_SECS(sbi
), hint
);
2700 if (secno
>= MAIN_SECS(sbi
)) {
2701 secno
= find_first_zero_bit(free_i
->free_secmap
,
2703 if (secno
>= MAIN_SECS(sbi
)) {
2708 segno
= GET_SEG_FROM_SEC(sbi
, secno
);
2709 zoneno
= GET_ZONE_FROM_SEC(sbi
, secno
);
2711 /* give up on finding another zone */
2714 if (sbi
->secs_per_zone
== 1)
2716 if (zoneno
== old_zoneno
)
2718 for (i
= 0; i
< NR_CURSEG_TYPE
; i
++)
2719 if (CURSEG_I(sbi
, i
)->zone
== zoneno
)
2722 if (i
< NR_CURSEG_TYPE
) {
2723 /* zone is in user, try another */
2724 if (zoneno
+ 1 >= total_zones
)
2727 hint
= (zoneno
+ 1) * sbi
->secs_per_zone
;
2729 goto find_other_zone
;
2732 /* set it as dirty segment in free segmap */
2733 f2fs_bug_on(sbi
, test_bit(segno
, free_i
->free_segmap
));
2735 /* no free section in conventional zone */
2736 if (new_sec
&& pinning
&&
2737 !f2fs_valid_pinned_area(sbi
, START_BLOCK(sbi
, segno
))) {
2741 __set_inuse(sbi
, segno
);
2744 spin_unlock(&free_i
->segmap_lock
);
2746 if (ret
== -ENOSPC
) {
2747 f2fs_stop_checkpoint(sbi
, false, STOP_CP_REASON_NO_SEGMENT
);
2748 f2fs_bug_on(sbi
, 1);
2753 static void reset_curseg(struct f2fs_sb_info
*sbi
, int type
, int modified
)
2755 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
2756 struct summary_footer
*sum_footer
;
2757 unsigned short seg_type
= curseg
->seg_type
;
2759 /* only happen when get_new_segment() fails */
2760 if (curseg
->next_segno
== NULL_SEGNO
)
2763 curseg
->inited
= true;
2764 curseg
->segno
= curseg
->next_segno
;
2765 curseg
->zone
= GET_ZONE_FROM_SEG(sbi
, curseg
->segno
);
2766 curseg
->next_blkoff
= 0;
2767 curseg
->next_segno
= NULL_SEGNO
;
2769 sum_footer
= &(curseg
->sum_blk
->footer
);
2770 memset(sum_footer
, 0, sizeof(struct summary_footer
));
2772 sanity_check_seg_type(sbi
, seg_type
);
2774 if (IS_DATASEG(seg_type
))
2775 SET_SUM_TYPE(sum_footer
, SUM_TYPE_DATA
);
2776 if (IS_NODESEG(seg_type
))
2777 SET_SUM_TYPE(sum_footer
, SUM_TYPE_NODE
);
2778 __set_sit_entry_type(sbi
, seg_type
, curseg
->segno
, modified
);
2781 static unsigned int __get_next_segno(struct f2fs_sb_info
*sbi
, int type
)
2783 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
2784 unsigned short seg_type
= curseg
->seg_type
;
2786 sanity_check_seg_type(sbi
, seg_type
);
2787 if (__is_large_section(sbi
)) {
2788 if (f2fs_need_rand_seg(sbi
)) {
2789 unsigned int hint
= GET_SEC_FROM_SEG(sbi
, curseg
->segno
);
2791 if (GET_SEC_FROM_SEG(sbi
, curseg
->segno
+ 1) != hint
)
2792 return curseg
->segno
;
2793 return get_random_u32_inclusive(curseg
->segno
+ 1,
2794 GET_SEG_FROM_SEC(sbi
, hint
+ 1) - 1);
2796 return curseg
->segno
;
2797 } else if (f2fs_need_rand_seg(sbi
)) {
2798 return get_random_u32_below(MAIN_SECS(sbi
) * SEGS_PER_SEC(sbi
));
2801 /* inmem log may not locate on any segment after mount */
2802 if (!curseg
->inited
)
2805 if (unlikely(is_sbi_flag_set(sbi
, SBI_CP_DISABLED
)))
2808 if (seg_type
== CURSEG_HOT_DATA
|| IS_NODESEG(seg_type
))
2811 if (SIT_I(sbi
)->last_victim
[ALLOC_NEXT
])
2812 return SIT_I(sbi
)->last_victim
[ALLOC_NEXT
];
2814 /* find segments from 0 to reuse freed segments */
2815 if (F2FS_OPTION(sbi
).alloc_mode
== ALLOC_MODE_REUSE
)
2818 return curseg
->segno
;
2822 * Allocate a current working segment.
2823 * This function always allocates a free segment in LFS manner.
2825 static int new_curseg(struct f2fs_sb_info
*sbi
, int type
, bool new_sec
)
2827 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
2828 unsigned int segno
= curseg
->segno
;
2829 bool pinning
= type
== CURSEG_COLD_DATA_PINNED
;
2833 write_sum_page(sbi
, curseg
->sum_blk
, GET_SUM_BLOCK(sbi
, segno
));
2835 segno
= __get_next_segno(sbi
, type
);
2836 ret
= get_new_segment(sbi
, &segno
, new_sec
, pinning
);
2839 curseg
->segno
= NULL_SEGNO
;
2843 curseg
->next_segno
= segno
;
2844 reset_curseg(sbi
, type
, 1);
2845 curseg
->alloc_type
= LFS
;
2846 if (F2FS_OPTION(sbi
).fs_mode
== FS_MODE_FRAGMENT_BLK
)
2847 curseg
->fragment_remained_chunk
=
2848 get_random_u32_inclusive(1, sbi
->max_fragment_chunk
);
2852 static int __next_free_blkoff(struct f2fs_sb_info
*sbi
,
2853 int segno
, block_t start
)
2855 struct seg_entry
*se
= get_seg_entry(sbi
, segno
);
2856 int entries
= SIT_VBLOCK_MAP_SIZE
/ sizeof(unsigned long);
2857 unsigned long *target_map
= SIT_I(sbi
)->tmp_map
;
2858 unsigned long *ckpt_map
= (unsigned long *)se
->ckpt_valid_map
;
2859 unsigned long *cur_map
= (unsigned long *)se
->cur_valid_map
;
2862 for (i
= 0; i
< entries
; i
++)
2863 target_map
[i
] = ckpt_map
[i
] | cur_map
[i
];
2865 return __find_rev_next_zero_bit(target_map
, BLKS_PER_SEG(sbi
), start
);
2868 static int f2fs_find_next_ssr_block(struct f2fs_sb_info
*sbi
,
2869 struct curseg_info
*seg
)
2871 return __next_free_blkoff(sbi
, seg
->segno
, seg
->next_blkoff
+ 1);
2874 bool f2fs_segment_has_free_slot(struct f2fs_sb_info
*sbi
, int segno
)
2876 return __next_free_blkoff(sbi
, segno
, 0) < BLKS_PER_SEG(sbi
);
2880 * This function always allocates a used segment(from dirty seglist) by SSR
2881 * manner, so it should recover the existing segment information of valid blocks
2883 static int change_curseg(struct f2fs_sb_info
*sbi
, int type
)
2885 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
2886 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
2887 unsigned int new_segno
= curseg
->next_segno
;
2888 struct f2fs_summary_block
*sum_node
;
2889 struct page
*sum_page
;
2891 write_sum_page(sbi
, curseg
->sum_blk
, GET_SUM_BLOCK(sbi
, curseg
->segno
));
2893 __set_test_and_inuse(sbi
, new_segno
);
2895 mutex_lock(&dirty_i
->seglist_lock
);
2896 __remove_dirty_segment(sbi
, new_segno
, PRE
);
2897 __remove_dirty_segment(sbi
, new_segno
, DIRTY
);
2898 mutex_unlock(&dirty_i
->seglist_lock
);
2900 reset_curseg(sbi
, type
, 1);
2901 curseg
->alloc_type
= SSR
;
2902 curseg
->next_blkoff
= __next_free_blkoff(sbi
, curseg
->segno
, 0);
2904 sum_page
= f2fs_get_sum_page(sbi
, new_segno
);
2905 if (IS_ERR(sum_page
)) {
2906 /* GC won't be able to use stale summary pages by cp_error */
2907 memset(curseg
->sum_blk
, 0, SUM_ENTRY_SIZE
);
2908 return PTR_ERR(sum_page
);
2910 sum_node
= (struct f2fs_summary_block
*)page_address(sum_page
);
2911 memcpy(curseg
->sum_blk
, sum_node
, SUM_ENTRY_SIZE
);
2912 f2fs_put_page(sum_page
, 1);
2916 static int get_ssr_segment(struct f2fs_sb_info
*sbi
, int type
,
2917 int alloc_mode
, unsigned long long age
);
2919 static int get_atssr_segment(struct f2fs_sb_info
*sbi
, int type
,
2920 int target_type
, int alloc_mode
,
2921 unsigned long long age
)
2923 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
2926 curseg
->seg_type
= target_type
;
2928 if (get_ssr_segment(sbi
, type
, alloc_mode
, age
)) {
2929 struct seg_entry
*se
= get_seg_entry(sbi
, curseg
->next_segno
);
2931 curseg
->seg_type
= se
->type
;
2932 ret
= change_curseg(sbi
, type
);
2934 /* allocate cold segment by default */
2935 curseg
->seg_type
= CURSEG_COLD_DATA
;
2936 ret
= new_curseg(sbi
, type
, true);
2938 stat_inc_seg_type(sbi
, curseg
);
2942 static int __f2fs_init_atgc_curseg(struct f2fs_sb_info
*sbi
, bool force
)
2944 struct curseg_info
*curseg
= CURSEG_I(sbi
, CURSEG_ALL_DATA_ATGC
);
2947 if (!sbi
->am
.atgc_enabled
&& !force
)
2950 f2fs_down_read(&SM_I(sbi
)->curseg_lock
);
2952 mutex_lock(&curseg
->curseg_mutex
);
2953 down_write(&SIT_I(sbi
)->sentry_lock
);
2955 ret
= get_atssr_segment(sbi
, CURSEG_ALL_DATA_ATGC
,
2956 CURSEG_COLD_DATA
, SSR
, 0);
2958 up_write(&SIT_I(sbi
)->sentry_lock
);
2959 mutex_unlock(&curseg
->curseg_mutex
);
2961 f2fs_up_read(&SM_I(sbi
)->curseg_lock
);
2965 int f2fs_init_inmem_curseg(struct f2fs_sb_info
*sbi
)
2967 return __f2fs_init_atgc_curseg(sbi
, false);
2970 int f2fs_reinit_atgc_curseg(struct f2fs_sb_info
*sbi
)
2974 if (!test_opt(sbi
, ATGC
))
2976 if (sbi
->am
.atgc_enabled
)
2978 if (le64_to_cpu(F2FS_CKPT(sbi
)->elapsed_time
) <
2979 sbi
->am
.age_threshold
)
2982 ret
= __f2fs_init_atgc_curseg(sbi
, true);
2984 sbi
->am
.atgc_enabled
= true;
2985 f2fs_info(sbi
, "reenabled age threshold GC");
2990 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info
*sbi
, int type
)
2992 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
2994 mutex_lock(&curseg
->curseg_mutex
);
2995 if (!curseg
->inited
)
2998 if (get_valid_blocks(sbi
, curseg
->segno
, false)) {
2999 write_sum_page(sbi
, curseg
->sum_blk
,
3000 GET_SUM_BLOCK(sbi
, curseg
->segno
));
3002 mutex_lock(&DIRTY_I(sbi
)->seglist_lock
);
3003 __set_test_and_free(sbi
, curseg
->segno
, true);
3004 mutex_unlock(&DIRTY_I(sbi
)->seglist_lock
);
3007 mutex_unlock(&curseg
->curseg_mutex
);
3010 void f2fs_save_inmem_curseg(struct f2fs_sb_info
*sbi
)
3012 __f2fs_save_inmem_curseg(sbi
, CURSEG_COLD_DATA_PINNED
);
3014 if (sbi
->am
.atgc_enabled
)
3015 __f2fs_save_inmem_curseg(sbi
, CURSEG_ALL_DATA_ATGC
);
3018 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info
*sbi
, int type
)
3020 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
3022 mutex_lock(&curseg
->curseg_mutex
);
3023 if (!curseg
->inited
)
3025 if (get_valid_blocks(sbi
, curseg
->segno
, false))
3028 mutex_lock(&DIRTY_I(sbi
)->seglist_lock
);
3029 __set_test_and_inuse(sbi
, curseg
->segno
);
3030 mutex_unlock(&DIRTY_I(sbi
)->seglist_lock
);
3032 mutex_unlock(&curseg
->curseg_mutex
);
3035 void f2fs_restore_inmem_curseg(struct f2fs_sb_info
*sbi
)
3037 __f2fs_restore_inmem_curseg(sbi
, CURSEG_COLD_DATA_PINNED
);
3039 if (sbi
->am
.atgc_enabled
)
3040 __f2fs_restore_inmem_curseg(sbi
, CURSEG_ALL_DATA_ATGC
);
3043 static int get_ssr_segment(struct f2fs_sb_info
*sbi
, int type
,
3044 int alloc_mode
, unsigned long long age
)
3046 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
3047 unsigned segno
= NULL_SEGNO
;
3048 unsigned short seg_type
= curseg
->seg_type
;
3050 bool reversed
= false;
3052 sanity_check_seg_type(sbi
, seg_type
);
3054 /* f2fs_need_SSR() already forces to do this */
3055 if (!f2fs_get_victim(sbi
, &segno
, BG_GC
, seg_type
, alloc_mode
, age
)) {
3056 curseg
->next_segno
= segno
;
3060 /* For node segments, let's do SSR more intensively */
3061 if (IS_NODESEG(seg_type
)) {
3062 if (seg_type
>= CURSEG_WARM_NODE
) {
3064 i
= CURSEG_COLD_NODE
;
3066 i
= CURSEG_HOT_NODE
;
3068 cnt
= NR_CURSEG_NODE_TYPE
;
3070 if (seg_type
>= CURSEG_WARM_DATA
) {
3072 i
= CURSEG_COLD_DATA
;
3074 i
= CURSEG_HOT_DATA
;
3076 cnt
= NR_CURSEG_DATA_TYPE
;
3079 for (; cnt
-- > 0; reversed
? i
-- : i
++) {
3082 if (!f2fs_get_victim(sbi
, &segno
, BG_GC
, i
, alloc_mode
, age
)) {
3083 curseg
->next_segno
= segno
;
3088 /* find valid_blocks=0 in dirty list */
3089 if (unlikely(is_sbi_flag_set(sbi
, SBI_CP_DISABLED
))) {
3090 segno
= get_free_segment(sbi
);
3091 if (segno
!= NULL_SEGNO
) {
3092 curseg
->next_segno
= segno
;
3099 static bool need_new_seg(struct f2fs_sb_info
*sbi
, int type
)
3101 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
3103 if (!is_set_ckpt_flags(sbi
, CP_CRC_RECOVERY_FLAG
) &&
3104 curseg
->seg_type
== CURSEG_WARM_NODE
)
3106 if (curseg
->alloc_type
== LFS
&& is_next_segment_free(sbi
, curseg
) &&
3107 likely(!is_sbi_flag_set(sbi
, SBI_CP_DISABLED
)))
3109 if (!f2fs_need_SSR(sbi
) || !get_ssr_segment(sbi
, type
, SSR
, 0))
3114 int f2fs_allocate_segment_for_resize(struct f2fs_sb_info
*sbi
, int type
,
3115 unsigned int start
, unsigned int end
)
3117 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
3121 f2fs_down_read(&SM_I(sbi
)->curseg_lock
);
3122 mutex_lock(&curseg
->curseg_mutex
);
3123 down_write(&SIT_I(sbi
)->sentry_lock
);
3125 segno
= CURSEG_I(sbi
, type
)->segno
;
3126 if (segno
< start
|| segno
> end
)
3129 if (f2fs_need_SSR(sbi
) && get_ssr_segment(sbi
, type
, SSR
, 0))
3130 ret
= change_curseg(sbi
, type
);
3132 ret
= new_curseg(sbi
, type
, true);
3134 stat_inc_seg_type(sbi
, curseg
);
3136 locate_dirty_segment(sbi
, segno
);
3138 up_write(&SIT_I(sbi
)->sentry_lock
);
3140 if (segno
!= curseg
->segno
)
3141 f2fs_notice(sbi
, "For resize: curseg of type %d: %u ==> %u",
3142 type
, segno
, curseg
->segno
);
3144 mutex_unlock(&curseg
->curseg_mutex
);
3145 f2fs_up_read(&SM_I(sbi
)->curseg_lock
);
3149 static int __allocate_new_segment(struct f2fs_sb_info
*sbi
, int type
,
3150 bool new_sec
, bool force
)
3152 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
3153 unsigned int old_segno
;
3156 if (type
== CURSEG_COLD_DATA_PINNED
&& !curseg
->inited
)
3159 if (!force
&& curseg
->inited
&&
3160 !curseg
->next_blkoff
&&
3161 !get_valid_blocks(sbi
, curseg
->segno
, new_sec
) &&
3162 !get_ckpt_valid_blocks(sbi
, curseg
->segno
, new_sec
))
3166 old_segno
= curseg
->segno
;
3167 err
= new_curseg(sbi
, type
, true);
3170 stat_inc_seg_type(sbi
, curseg
);
3171 locate_dirty_segment(sbi
, old_segno
);
3175 int f2fs_allocate_new_section(struct f2fs_sb_info
*sbi
, int type
, bool force
)
3179 f2fs_down_read(&SM_I(sbi
)->curseg_lock
);
3180 down_write(&SIT_I(sbi
)->sentry_lock
);
3181 ret
= __allocate_new_segment(sbi
, type
, true, force
);
3182 up_write(&SIT_I(sbi
)->sentry_lock
);
3183 f2fs_up_read(&SM_I(sbi
)->curseg_lock
);
3188 int f2fs_allocate_pinning_section(struct f2fs_sb_info
*sbi
)
3191 bool gc_required
= true;
3195 err
= f2fs_allocate_new_section(sbi
, CURSEG_COLD_DATA_PINNED
, false);
3196 f2fs_unlock_op(sbi
);
3198 if (f2fs_sb_has_blkzoned(sbi
) && err
== -EAGAIN
&& gc_required
) {
3199 f2fs_down_write(&sbi
->gc_lock
);
3200 err
= f2fs_gc_range(sbi
, 0, GET_SEGNO(sbi
, FDEV(0).end_blk
), true, 1);
3201 f2fs_up_write(&sbi
->gc_lock
);
3203 gc_required
= false;
3211 int f2fs_allocate_new_segments(struct f2fs_sb_info
*sbi
)
3216 f2fs_down_read(&SM_I(sbi
)->curseg_lock
);
3217 down_write(&SIT_I(sbi
)->sentry_lock
);
3218 for (i
= CURSEG_HOT_DATA
; i
<= CURSEG_COLD_DATA
; i
++)
3219 err
+= __allocate_new_segment(sbi
, i
, false, false);
3220 up_write(&SIT_I(sbi
)->sentry_lock
);
3221 f2fs_up_read(&SM_I(sbi
)->curseg_lock
);
3226 bool f2fs_exist_trim_candidates(struct f2fs_sb_info
*sbi
,
3227 struct cp_control
*cpc
)
3229 __u64 trim_start
= cpc
->trim_start
;
3230 bool has_candidate
= false;
3232 down_write(&SIT_I(sbi
)->sentry_lock
);
3233 for (; cpc
->trim_start
<= cpc
->trim_end
; cpc
->trim_start
++) {
3234 if (add_discard_addrs(sbi
, cpc
, true)) {
3235 has_candidate
= true;
3239 up_write(&SIT_I(sbi
)->sentry_lock
);
3241 cpc
->trim_start
= trim_start
;
3242 return has_candidate
;
3245 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info
*sbi
,
3246 struct discard_policy
*dpolicy
,
3247 unsigned int start
, unsigned int end
)
3249 struct discard_cmd_control
*dcc
= SM_I(sbi
)->dcc_info
;
3250 struct discard_cmd
*prev_dc
= NULL
, *next_dc
= NULL
;
3251 struct rb_node
**insert_p
= NULL
, *insert_parent
= NULL
;
3252 struct discard_cmd
*dc
;
3253 struct blk_plug plug
;
3255 unsigned int trimmed
= 0;
3260 mutex_lock(&dcc
->cmd_lock
);
3261 if (unlikely(dcc
->rbtree_check
))
3262 f2fs_bug_on(sbi
, !f2fs_check_discard_tree(sbi
));
3264 dc
= __lookup_discard_cmd_ret(&dcc
->root
, start
,
3265 &prev_dc
, &next_dc
, &insert_p
, &insert_parent
);
3269 blk_start_plug(&plug
);
3271 while (dc
&& dc
->di
.lstart
<= end
) {
3272 struct rb_node
*node
;
3275 if (dc
->di
.len
< dpolicy
->granularity
)
3278 if (dc
->state
!= D_PREP
) {
3279 list_move_tail(&dc
->list
, &dcc
->fstrim_list
);
3283 err
= __submit_discard_cmd(sbi
, dpolicy
, dc
, &issued
);
3285 if (issued
>= dpolicy
->max_requests
) {
3286 start
= dc
->di
.lstart
+ dc
->di
.len
;
3289 __remove_discard_cmd(sbi
, dc
);
3291 blk_finish_plug(&plug
);
3292 mutex_unlock(&dcc
->cmd_lock
);
3293 trimmed
+= __wait_all_discard_cmd(sbi
, NULL
);
3294 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT
);
3298 node
= rb_next(&dc
->rb_node
);
3300 __remove_discard_cmd(sbi
, dc
);
3301 dc
= rb_entry_safe(node
, struct discard_cmd
, rb_node
);
3303 if (fatal_signal_pending(current
))
3307 blk_finish_plug(&plug
);
3308 mutex_unlock(&dcc
->cmd_lock
);
3313 int f2fs_trim_fs(struct f2fs_sb_info
*sbi
, struct fstrim_range
*range
)
3315 __u64 start
= F2FS_BYTES_TO_BLK(range
->start
);
3316 __u64 end
= start
+ F2FS_BYTES_TO_BLK(range
->len
) - 1;
3317 unsigned int start_segno
, end_segno
;
3318 block_t start_block
, end_block
;
3319 struct cp_control cpc
;
3320 struct discard_policy dpolicy
;
3321 unsigned long long trimmed
= 0;
3323 bool need_align
= f2fs_lfs_mode(sbi
) && __is_large_section(sbi
);
3325 if (start
>= MAX_BLKADDR(sbi
) || range
->len
< sbi
->blocksize
)
3328 if (end
< MAIN_BLKADDR(sbi
))
3331 if (is_sbi_flag_set(sbi
, SBI_NEED_FSCK
)) {
3332 f2fs_warn(sbi
, "Found FS corruption, run fsck to fix.");
3333 return -EFSCORRUPTED
;
3336 /* start/end segment number in main_area */
3337 start_segno
= (start
<= MAIN_BLKADDR(sbi
)) ? 0 : GET_SEGNO(sbi
, start
);
3338 end_segno
= (end
>= MAX_BLKADDR(sbi
)) ? MAIN_SEGS(sbi
) - 1 :
3339 GET_SEGNO(sbi
, end
);
3341 start_segno
= rounddown(start_segno
, SEGS_PER_SEC(sbi
));
3342 end_segno
= roundup(end_segno
+ 1, SEGS_PER_SEC(sbi
)) - 1;
3345 cpc
.reason
= CP_DISCARD
;
3346 cpc
.trim_minlen
= max_t(__u64
, 1, F2FS_BYTES_TO_BLK(range
->minlen
));
3347 cpc
.trim_start
= start_segno
;
3348 cpc
.trim_end
= end_segno
;
3350 if (sbi
->discard_blks
== 0)
3353 f2fs_down_write(&sbi
->gc_lock
);
3354 stat_inc_cp_call_count(sbi
, TOTAL_CALL
);
3355 err
= f2fs_write_checkpoint(sbi
, &cpc
);
3356 f2fs_up_write(&sbi
->gc_lock
);
3361 * We filed discard candidates, but actually we don't need to wait for
3362 * all of them, since they'll be issued in idle time along with runtime
3363 * discard option. User configuration looks like using runtime discard
3364 * or periodic fstrim instead of it.
3366 if (f2fs_realtime_discard_enable(sbi
))
3369 start_block
= START_BLOCK(sbi
, start_segno
);
3370 end_block
= START_BLOCK(sbi
, end_segno
+ 1);
3372 __init_discard_policy(sbi
, &dpolicy
, DPOLICY_FSTRIM
, cpc
.trim_minlen
);
3373 trimmed
= __issue_discard_cmd_range(sbi
, &dpolicy
,
3374 start_block
, end_block
);
3376 trimmed
+= __wait_discard_cmd_range(sbi
, &dpolicy
,
3377 start_block
, end_block
);
3380 range
->len
= F2FS_BLK_TO_BYTES(trimmed
);
3384 int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info
*sbi
, enum rw_hint hint
)
3386 if (F2FS_OPTION(sbi
).active_logs
== 2)
3387 return CURSEG_HOT_DATA
;
3388 else if (F2FS_OPTION(sbi
).active_logs
== 4)
3389 return CURSEG_COLD_DATA
;
3391 /* active_log == 6 */
3393 case WRITE_LIFE_SHORT
:
3394 return CURSEG_HOT_DATA
;
3395 case WRITE_LIFE_EXTREME
:
3396 return CURSEG_COLD_DATA
;
3398 return CURSEG_WARM_DATA
;
3403 * This returns write hints for each segment type. This hints will be
3404 * passed down to block layer as below by default.
3408 * META WRITE_LIFE_NONE|REQ_META
3409 * HOT_NODE WRITE_LIFE_NONE
3410 * WARM_NODE WRITE_LIFE_MEDIUM
3411 * COLD_NODE WRITE_LIFE_LONG
3412 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME
3413 * extension list " "
3416 * COLD_DATA WRITE_LIFE_EXTREME
3417 * HOT_DATA WRITE_LIFE_SHORT
3418 * WARM_DATA WRITE_LIFE_NOT_SET
3421 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
3422 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
3423 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
3424 * WRITE_LIFE_NONE " WRITE_LIFE_NONE
3425 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM
3426 * WRITE_LIFE_LONG " WRITE_LIFE_LONG
3428 enum rw_hint
f2fs_io_type_to_rw_hint(struct f2fs_sb_info
*sbi
,
3429 enum page_type type
, enum temp_type temp
)
3435 return WRITE_LIFE_NOT_SET
;
3437 return WRITE_LIFE_SHORT
;
3439 return WRITE_LIFE_EXTREME
;
3441 return WRITE_LIFE_NONE
;
3446 return WRITE_LIFE_MEDIUM
;
3448 return WRITE_LIFE_NONE
;
3450 return WRITE_LIFE_LONG
;
3452 return WRITE_LIFE_NONE
;
3455 return WRITE_LIFE_NONE
;
3457 return WRITE_LIFE_NONE
;
3461 static int __get_segment_type_2(struct f2fs_io_info
*fio
)
3463 if (fio
->type
== DATA
)
3464 return CURSEG_HOT_DATA
;
3466 return CURSEG_HOT_NODE
;
3469 static int __get_segment_type_4(struct f2fs_io_info
*fio
)
3471 if (fio
->type
== DATA
) {
3472 struct inode
*inode
= fio
->page
->mapping
->host
;
3474 if (S_ISDIR(inode
->i_mode
))
3475 return CURSEG_HOT_DATA
;
3477 return CURSEG_COLD_DATA
;
3479 if (IS_DNODE(fio
->page
) && is_cold_node(fio
->page
))
3480 return CURSEG_WARM_NODE
;
3482 return CURSEG_COLD_NODE
;
3486 static int __get_age_segment_type(struct inode
*inode
, pgoff_t pgofs
)
3488 struct f2fs_sb_info
*sbi
= F2FS_I_SB(inode
);
3489 struct extent_info ei
= {};
3491 if (f2fs_lookup_age_extent_cache(inode
, pgofs
, &ei
)) {
3493 return NO_CHECK_TYPE
;
3494 if (ei
.age
<= sbi
->hot_data_age_threshold
)
3495 return CURSEG_HOT_DATA
;
3496 if (ei
.age
<= sbi
->warm_data_age_threshold
)
3497 return CURSEG_WARM_DATA
;
3498 return CURSEG_COLD_DATA
;
3500 return NO_CHECK_TYPE
;
3503 static int __get_segment_type_6(struct f2fs_io_info
*fio
)
3505 if (fio
->type
== DATA
) {
3506 struct inode
*inode
= fio
->page
->mapping
->host
;
3509 if (is_inode_flag_set(inode
, FI_ALIGNED_WRITE
))
3510 return CURSEG_COLD_DATA_PINNED
;
3512 if (page_private_gcing(fio
->page
)) {
3513 if (fio
->sbi
->am
.atgc_enabled
&&
3514 (fio
->io_type
== FS_DATA_IO
) &&
3515 (fio
->sbi
->gc_mode
!= GC_URGENT_HIGH
) &&
3516 __is_valid_data_blkaddr(fio
->old_blkaddr
) &&
3517 !is_inode_flag_set(inode
, FI_OPU_WRITE
))
3518 return CURSEG_ALL_DATA_ATGC
;
3520 return CURSEG_COLD_DATA
;
3522 if (file_is_cold(inode
) || f2fs_need_compress_data(inode
))
3523 return CURSEG_COLD_DATA
;
3525 type
= __get_age_segment_type(inode
, fio
->page
->index
);
3526 if (type
!= NO_CHECK_TYPE
)
3529 if (file_is_hot(inode
) ||
3530 is_inode_flag_set(inode
, FI_HOT_DATA
) ||
3531 f2fs_is_cow_file(inode
))
3532 return CURSEG_HOT_DATA
;
3533 return f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode
),
3534 inode
->i_write_hint
);
3536 if (IS_DNODE(fio
->page
))
3537 return is_cold_node(fio
->page
) ? CURSEG_WARM_NODE
:
3539 return CURSEG_COLD_NODE
;
3543 int f2fs_get_segment_temp(int seg_type
)
3545 if (IS_HOT(seg_type
))
3547 else if (IS_WARM(seg_type
))
3552 static int __get_segment_type(struct f2fs_io_info
*fio
)
3556 switch (F2FS_OPTION(fio
->sbi
).active_logs
) {
3558 type
= __get_segment_type_2(fio
);
3561 type
= __get_segment_type_4(fio
);
3564 type
= __get_segment_type_6(fio
);
3567 f2fs_bug_on(fio
->sbi
, true);
3570 fio
->temp
= f2fs_get_segment_temp(type
);
3575 static void f2fs_randomize_chunk(struct f2fs_sb_info
*sbi
,
3576 struct curseg_info
*seg
)
3578 /* To allocate block chunks in different sizes, use random number */
3579 if (--seg
->fragment_remained_chunk
> 0)
3582 seg
->fragment_remained_chunk
=
3583 get_random_u32_inclusive(1, sbi
->max_fragment_chunk
);
3585 get_random_u32_inclusive(1, sbi
->max_fragment_hole
);
3588 static void reset_curseg_fields(struct curseg_info
*curseg
)
3590 curseg
->inited
= false;
3591 curseg
->segno
= NULL_SEGNO
;
3592 curseg
->next_segno
= 0;
3595 int f2fs_allocate_data_block(struct f2fs_sb_info
*sbi
, struct page
*page
,
3596 block_t old_blkaddr
, block_t
*new_blkaddr
,
3597 struct f2fs_summary
*sum
, int type
,
3598 struct f2fs_io_info
*fio
)
3600 struct sit_info
*sit_i
= SIT_I(sbi
);
3601 struct curseg_info
*curseg
= CURSEG_I(sbi
, type
);
3602 unsigned long long old_mtime
;
3603 bool from_gc
= (type
== CURSEG_ALL_DATA_ATGC
);
3604 struct seg_entry
*se
= NULL
;
3605 bool segment_full
= false;
3608 f2fs_down_read(&SM_I(sbi
)->curseg_lock
);
3610 mutex_lock(&curseg
->curseg_mutex
);
3611 down_write(&sit_i
->sentry_lock
);
3613 if (curseg
->segno
== NULL_SEGNO
) {
3619 f2fs_bug_on(sbi
, GET_SEGNO(sbi
, old_blkaddr
) == NULL_SEGNO
);
3620 se
= get_seg_entry(sbi
, GET_SEGNO(sbi
, old_blkaddr
));
3621 sanity_check_seg_type(sbi
, se
->type
);
3622 f2fs_bug_on(sbi
, IS_NODESEG(se
->type
));
3624 *new_blkaddr
= NEXT_FREE_BLKADDR(sbi
, curseg
);
3626 f2fs_bug_on(sbi
, curseg
->next_blkoff
>= BLKS_PER_SEG(sbi
));
3628 f2fs_wait_discard_bio(sbi
, *new_blkaddr
);
3630 curseg
->sum_blk
->entries
[curseg
->next_blkoff
] = *sum
;
3631 if (curseg
->alloc_type
== SSR
) {
3632 curseg
->next_blkoff
= f2fs_find_next_ssr_block(sbi
, curseg
);
3634 curseg
->next_blkoff
++;
3635 if (F2FS_OPTION(sbi
).fs_mode
== FS_MODE_FRAGMENT_BLK
)
3636 f2fs_randomize_chunk(sbi
, curseg
);
3638 if (curseg
->next_blkoff
>= f2fs_usable_blks_in_seg(sbi
, curseg
->segno
))
3639 segment_full
= true;
3640 stat_inc_block_count(sbi
, curseg
);
3643 old_mtime
= get_segment_mtime(sbi
, old_blkaddr
);
3645 update_segment_mtime(sbi
, old_blkaddr
, 0);
3648 update_segment_mtime(sbi
, *new_blkaddr
, old_mtime
);
3651 * SIT information should be updated before segment allocation,
3652 * since SSR needs latest valid block information.
3654 update_sit_entry(sbi
, *new_blkaddr
, 1);
3655 update_sit_entry(sbi
, old_blkaddr
, -1);
3658 * If the current segment is full, flush it out and replace it with a
3662 if (type
== CURSEG_COLD_DATA_PINNED
&&
3663 !((curseg
->segno
+ 1) % sbi
->segs_per_sec
)) {
3664 write_sum_page(sbi
, curseg
->sum_blk
,
3665 GET_SUM_BLOCK(sbi
, curseg
->segno
));
3666 reset_curseg_fields(curseg
);
3667 goto skip_new_segment
;
3671 ret
= get_atssr_segment(sbi
, type
, se
->type
,
3674 if (need_new_seg(sbi
, type
))
3675 ret
= new_curseg(sbi
, type
, false);
3677 ret
= change_curseg(sbi
, type
);
3678 stat_inc_seg_type(sbi
, curseg
);
3687 * segment dirty status should be updated after segment allocation,
3688 * so we just need to update status only one time after previous
3689 * segment being closed.
3691 locate_dirty_segment(sbi
, GET_SEGNO(sbi
, old_blkaddr
));
3692 locate_dirty_segment(sbi
, GET_SEGNO(sbi
, *new_blkaddr
));
3694 if (IS_DATASEG(curseg
->seg_type
))
3695 atomic64_inc(&sbi
->allocated_data_blocks
);
3697 up_write(&sit_i
->sentry_lock
);
3699 if (page
&& IS_NODESEG(curseg
->seg_type
)) {
3700 fill_node_footer_blkaddr(page
, NEXT_FREE_BLKADDR(sbi
, curseg
));
3702 f2fs_inode_chksum_set(sbi
, page
);
3706 struct f2fs_bio_info
*io
;
3708 INIT_LIST_HEAD(&fio
->list
);
3710 io
= sbi
->write_io
[fio
->type
] + fio
->temp
;
3711 spin_lock(&io
->io_lock
);
3712 list_add_tail(&fio
->list
, &io
->io_list
);
3713 spin_unlock(&io
->io_lock
);
3716 mutex_unlock(&curseg
->curseg_mutex
);
3717 f2fs_up_read(&SM_I(sbi
)->curseg_lock
);
3721 *new_blkaddr
= NULL_ADDR
;
3722 up_write(&sit_i
->sentry_lock
);
3723 mutex_unlock(&curseg
->curseg_mutex
);
3724 f2fs_up_read(&SM_I(sbi
)->curseg_lock
);
3728 void f2fs_update_device_state(struct f2fs_sb_info
*sbi
, nid_t ino
,
3729 block_t blkaddr
, unsigned int blkcnt
)
3731 if (!f2fs_is_multi_device(sbi
))
3735 unsigned int devidx
= f2fs_target_device_index(sbi
, blkaddr
);
3736 unsigned int blks
= FDEV(devidx
).end_blk
- blkaddr
+ 1;
3738 /* update device state for fsync */
3739 f2fs_set_dirty_device(sbi
, ino
, devidx
, FLUSH_INO
);
3741 /* update device state for checkpoint */
3742 if (!f2fs_test_bit(devidx
, (char *)&sbi
->dirty_device
)) {
3743 spin_lock(&sbi
->dev_lock
);
3744 f2fs_set_bit(devidx
, (char *)&sbi
->dirty_device
);
3745 spin_unlock(&sbi
->dev_lock
);
3755 static void do_write_page(struct f2fs_summary
*sum
, struct f2fs_io_info
*fio
)
3757 int type
= __get_segment_type(fio
);
3758 bool keep_order
= (f2fs_lfs_mode(fio
->sbi
) && type
== CURSEG_COLD_DATA
);
3761 f2fs_down_read(&fio
->sbi
->io_order_lock
);
3763 if (f2fs_allocate_data_block(fio
->sbi
, fio
->page
, fio
->old_blkaddr
,
3764 &fio
->new_blkaddr
, sum
, type
, fio
)) {
3765 if (fscrypt_inode_uses_fs_layer_crypto(fio
->page
->mapping
->host
))
3766 fscrypt_finalize_bounce_page(&fio
->encrypted_page
);
3767 end_page_writeback(fio
->page
);
3768 if (f2fs_in_warm_node_list(fio
->sbi
, fio
->page
))
3769 f2fs_del_fsync_node_entry(fio
->sbi
, fio
->page
);
3772 if (GET_SEGNO(fio
->sbi
, fio
->old_blkaddr
) != NULL_SEGNO
)
3773 f2fs_invalidate_internal_cache(fio
->sbi
, fio
->old_blkaddr
);
3775 /* writeout dirty page into bdev */
3776 f2fs_submit_page_write(fio
);
3778 f2fs_update_device_state(fio
->sbi
, fio
->ino
, fio
->new_blkaddr
, 1);
3781 f2fs_up_read(&fio
->sbi
->io_order_lock
);
3784 void f2fs_do_write_meta_page(struct f2fs_sb_info
*sbi
, struct page
*page
,
3785 enum iostat_type io_type
)
3787 struct f2fs_io_info fio
= {
3792 .op_flags
= REQ_SYNC
| REQ_META
| REQ_PRIO
,
3793 .old_blkaddr
= page
->index
,
3794 .new_blkaddr
= page
->index
,
3796 .encrypted_page
= NULL
,
3800 if (unlikely(page
->index
>= MAIN_BLKADDR(sbi
)))
3801 fio
.op_flags
&= ~REQ_META
;
3803 set_page_writeback(page
);
3804 f2fs_submit_page_write(&fio
);
3806 stat_inc_meta_count(sbi
, page
->index
);
3807 f2fs_update_iostat(sbi
, NULL
, io_type
, F2FS_BLKSIZE
);
3810 void f2fs_do_write_node_page(unsigned int nid
, struct f2fs_io_info
*fio
)
3812 struct f2fs_summary sum
;
3814 set_summary(&sum
, nid
, 0, 0);
3815 do_write_page(&sum
, fio
);
3817 f2fs_update_iostat(fio
->sbi
, NULL
, fio
->io_type
, F2FS_BLKSIZE
);
3820 void f2fs_outplace_write_data(struct dnode_of_data
*dn
,
3821 struct f2fs_io_info
*fio
)
3823 struct f2fs_sb_info
*sbi
= fio
->sbi
;
3824 struct f2fs_summary sum
;
3826 f2fs_bug_on(sbi
, dn
->data_blkaddr
== NULL_ADDR
);
3827 if (fio
->io_type
== FS_DATA_IO
|| fio
->io_type
== FS_CP_DATA_IO
)
3828 f2fs_update_age_extent_cache(dn
);
3829 set_summary(&sum
, dn
->nid
, dn
->ofs_in_node
, fio
->version
);
3830 do_write_page(&sum
, fio
);
3831 f2fs_update_data_blkaddr(dn
, fio
->new_blkaddr
);
3833 f2fs_update_iostat(sbi
, dn
->inode
, fio
->io_type
, F2FS_BLKSIZE
);
3836 int f2fs_inplace_write_data(struct f2fs_io_info
*fio
)
3839 struct f2fs_sb_info
*sbi
= fio
->sbi
;
3842 fio
->new_blkaddr
= fio
->old_blkaddr
;
3843 /* i/o temperature is needed for passing down write hints */
3844 __get_segment_type(fio
);
3846 segno
= GET_SEGNO(sbi
, fio
->new_blkaddr
);
3848 if (!IS_DATASEG(get_seg_entry(sbi
, segno
)->type
)) {
3849 set_sbi_flag(sbi
, SBI_NEED_FSCK
);
3850 f2fs_warn(sbi
, "%s: incorrect segment(%u) type, run fsck to fix.",
3852 err
= -EFSCORRUPTED
;
3853 f2fs_handle_error(sbi
, ERROR_INCONSISTENT_SUM_TYPE
);
3857 if (f2fs_cp_error(sbi
)) {
3863 f2fs_truncate_meta_inode_pages(sbi
, fio
->new_blkaddr
, 1);
3865 stat_inc_inplace_blocks(fio
->sbi
);
3867 if (fio
->bio
&& !IS_F2FS_IPU_NOCACHE(sbi
))
3868 err
= f2fs_merge_page_bio(fio
);
3870 err
= f2fs_submit_page_bio(fio
);
3872 f2fs_update_device_state(fio
->sbi
, fio
->ino
,
3873 fio
->new_blkaddr
, 1);
3874 f2fs_update_iostat(fio
->sbi
, fio
->page
->mapping
->host
,
3875 fio
->io_type
, F2FS_BLKSIZE
);
3880 if (fio
->bio
&& *(fio
->bio
)) {
3881 struct bio
*bio
= *(fio
->bio
);
3883 bio
->bi_status
= BLK_STS_IOERR
;
3890 static inline int __f2fs_get_curseg(struct f2fs_sb_info
*sbi
,
3895 for (i
= CURSEG_HOT_DATA
; i
< NO_CHECK_TYPE
; i
++) {
3896 if (CURSEG_I(sbi
, i
)->segno
== segno
)
3902 void f2fs_do_replace_block(struct f2fs_sb_info
*sbi
, struct f2fs_summary
*sum
,
3903 block_t old_blkaddr
, block_t new_blkaddr
,
3904 bool recover_curseg
, bool recover_newaddr
,
3907 struct sit_info
*sit_i
= SIT_I(sbi
);
3908 struct curseg_info
*curseg
;
3909 unsigned int segno
, old_cursegno
;
3910 struct seg_entry
*se
;
3912 unsigned short old_blkoff
;
3913 unsigned char old_alloc_type
;
3915 segno
= GET_SEGNO(sbi
, new_blkaddr
);
3916 se
= get_seg_entry(sbi
, segno
);
3919 f2fs_down_write(&SM_I(sbi
)->curseg_lock
);
3921 if (!recover_curseg
) {
3922 /* for recovery flow */
3923 if (se
->valid_blocks
== 0 && !IS_CURSEG(sbi
, segno
)) {
3924 if (old_blkaddr
== NULL_ADDR
)
3925 type
= CURSEG_COLD_DATA
;
3927 type
= CURSEG_WARM_DATA
;
3930 if (IS_CURSEG(sbi
, segno
)) {
3931 /* se->type is volatile as SSR allocation */
3932 type
= __f2fs_get_curseg(sbi
, segno
);
3933 f2fs_bug_on(sbi
, type
== NO_CHECK_TYPE
);
3935 type
= CURSEG_WARM_DATA
;
3939 f2fs_bug_on(sbi
, !IS_DATASEG(type
));
3940 curseg
= CURSEG_I(sbi
, type
);
3942 mutex_lock(&curseg
->curseg_mutex
);
3943 down_write(&sit_i
->sentry_lock
);
3945 old_cursegno
= curseg
->segno
;
3946 old_blkoff
= curseg
->next_blkoff
;
3947 old_alloc_type
= curseg
->alloc_type
;
3949 /* change the current segment */
3950 if (segno
!= curseg
->segno
) {
3951 curseg
->next_segno
= segno
;
3952 if (change_curseg(sbi
, type
))
3956 curseg
->next_blkoff
= GET_BLKOFF_FROM_SEG0(sbi
, new_blkaddr
);
3957 curseg
->sum_blk
->entries
[curseg
->next_blkoff
] = *sum
;
3959 if (!recover_curseg
|| recover_newaddr
) {
3961 update_segment_mtime(sbi
, new_blkaddr
, 0);
3962 update_sit_entry(sbi
, new_blkaddr
, 1);
3964 if (GET_SEGNO(sbi
, old_blkaddr
) != NULL_SEGNO
) {
3965 f2fs_invalidate_internal_cache(sbi
, old_blkaddr
);
3967 update_segment_mtime(sbi
, old_blkaddr
, 0);
3968 update_sit_entry(sbi
, old_blkaddr
, -1);
3971 locate_dirty_segment(sbi
, GET_SEGNO(sbi
, old_blkaddr
));
3972 locate_dirty_segment(sbi
, GET_SEGNO(sbi
, new_blkaddr
));
3974 locate_dirty_segment(sbi
, old_cursegno
);
3976 if (recover_curseg
) {
3977 if (old_cursegno
!= curseg
->segno
) {
3978 curseg
->next_segno
= old_cursegno
;
3979 if (change_curseg(sbi
, type
))
3982 curseg
->next_blkoff
= old_blkoff
;
3983 curseg
->alloc_type
= old_alloc_type
;
3987 up_write(&sit_i
->sentry_lock
);
3988 mutex_unlock(&curseg
->curseg_mutex
);
3989 f2fs_up_write(&SM_I(sbi
)->curseg_lock
);
3992 void f2fs_replace_block(struct f2fs_sb_info
*sbi
, struct dnode_of_data
*dn
,
3993 block_t old_addr
, block_t new_addr
,
3994 unsigned char version
, bool recover_curseg
,
3995 bool recover_newaddr
)
3997 struct f2fs_summary sum
;
3999 set_summary(&sum
, dn
->nid
, dn
->ofs_in_node
, version
);
4001 f2fs_do_replace_block(sbi
, &sum
, old_addr
, new_addr
,
4002 recover_curseg
, recover_newaddr
, false);
4004 f2fs_update_data_blkaddr(dn
, new_addr
);
4007 void f2fs_wait_on_page_writeback(struct page
*page
,
4008 enum page_type type
, bool ordered
, bool locked
)
4010 if (folio_test_writeback(page_folio(page
))) {
4011 struct f2fs_sb_info
*sbi
= F2FS_P_SB(page
);
4013 /* submit cached LFS IO */
4014 f2fs_submit_merged_write_cond(sbi
, NULL
, page
, 0, type
);
4015 /* submit cached IPU IO */
4016 f2fs_submit_merged_ipu_write(sbi
, NULL
, page
);
4018 wait_on_page_writeback(page
);
4019 f2fs_bug_on(sbi
, locked
&&
4020 folio_test_writeback(page_folio(page
)));
4022 wait_for_stable_page(page
);
4027 void f2fs_wait_on_block_writeback(struct inode
*inode
, block_t blkaddr
)
4029 struct f2fs_sb_info
*sbi
= F2FS_I_SB(inode
);
4032 if (!f2fs_meta_inode_gc_required(inode
))
4035 if (!__is_valid_data_blkaddr(blkaddr
))
4038 cpage
= find_lock_page(META_MAPPING(sbi
), blkaddr
);
4040 f2fs_wait_on_page_writeback(cpage
, DATA
, true, true);
4041 f2fs_put_page(cpage
, 1);
4045 void f2fs_wait_on_block_writeback_range(struct inode
*inode
, block_t blkaddr
,
4048 struct f2fs_sb_info
*sbi
= F2FS_I_SB(inode
);
4051 if (!f2fs_meta_inode_gc_required(inode
))
4054 for (i
= 0; i
< len
; i
++)
4055 f2fs_wait_on_block_writeback(inode
, blkaddr
+ i
);
4057 f2fs_truncate_meta_inode_pages(sbi
, blkaddr
, len
);
4060 static int read_compacted_summaries(struct f2fs_sb_info
*sbi
)
4062 struct f2fs_checkpoint
*ckpt
= F2FS_CKPT(sbi
);
4063 struct curseg_info
*seg_i
;
4064 unsigned char *kaddr
;
4069 start
= start_sum_block(sbi
);
4071 page
= f2fs_get_meta_page(sbi
, start
++);
4073 return PTR_ERR(page
);
4074 kaddr
= (unsigned char *)page_address(page
);
4076 /* Step 1: restore nat cache */
4077 seg_i
= CURSEG_I(sbi
, CURSEG_HOT_DATA
);
4078 memcpy(seg_i
->journal
, kaddr
, SUM_JOURNAL_SIZE
);
4080 /* Step 2: restore sit cache */
4081 seg_i
= CURSEG_I(sbi
, CURSEG_COLD_DATA
);
4082 memcpy(seg_i
->journal
, kaddr
+ SUM_JOURNAL_SIZE
, SUM_JOURNAL_SIZE
);
4083 offset
= 2 * SUM_JOURNAL_SIZE
;
4085 /* Step 3: restore summary entries */
4086 for (i
= CURSEG_HOT_DATA
; i
<= CURSEG_COLD_DATA
; i
++) {
4087 unsigned short blk_off
;
4090 seg_i
= CURSEG_I(sbi
, i
);
4091 segno
= le32_to_cpu(ckpt
->cur_data_segno
[i
]);
4092 blk_off
= le16_to_cpu(ckpt
->cur_data_blkoff
[i
]);
4093 seg_i
->next_segno
= segno
;
4094 reset_curseg(sbi
, i
, 0);
4095 seg_i
->alloc_type
= ckpt
->alloc_type
[i
];
4096 seg_i
->next_blkoff
= blk_off
;
4098 if (seg_i
->alloc_type
== SSR
)
4099 blk_off
= BLKS_PER_SEG(sbi
);
4101 for (j
= 0; j
< blk_off
; j
++) {
4102 struct f2fs_summary
*s
;
4104 s
= (struct f2fs_summary
*)(kaddr
+ offset
);
4105 seg_i
->sum_blk
->entries
[j
] = *s
;
4106 offset
+= SUMMARY_SIZE
;
4107 if (offset
+ SUMMARY_SIZE
<= PAGE_SIZE
-
4111 f2fs_put_page(page
, 1);
4114 page
= f2fs_get_meta_page(sbi
, start
++);
4116 return PTR_ERR(page
);
4117 kaddr
= (unsigned char *)page_address(page
);
4121 f2fs_put_page(page
, 1);
4125 static int read_normal_summaries(struct f2fs_sb_info
*sbi
, int type
)
4127 struct f2fs_checkpoint
*ckpt
= F2FS_CKPT(sbi
);
4128 struct f2fs_summary_block
*sum
;
4129 struct curseg_info
*curseg
;
4131 unsigned short blk_off
;
4132 unsigned int segno
= 0;
4133 block_t blk_addr
= 0;
4136 /* get segment number and block addr */
4137 if (IS_DATASEG(type
)) {
4138 segno
= le32_to_cpu(ckpt
->cur_data_segno
[type
]);
4139 blk_off
= le16_to_cpu(ckpt
->cur_data_blkoff
[type
-
4141 if (__exist_node_summaries(sbi
))
4142 blk_addr
= sum_blk_addr(sbi
, NR_CURSEG_PERSIST_TYPE
, type
);
4144 blk_addr
= sum_blk_addr(sbi
, NR_CURSEG_DATA_TYPE
, type
);
4146 segno
= le32_to_cpu(ckpt
->cur_node_segno
[type
-
4148 blk_off
= le16_to_cpu(ckpt
->cur_node_blkoff
[type
-
4150 if (__exist_node_summaries(sbi
))
4151 blk_addr
= sum_blk_addr(sbi
, NR_CURSEG_NODE_TYPE
,
4152 type
- CURSEG_HOT_NODE
);
4154 blk_addr
= GET_SUM_BLOCK(sbi
, segno
);
4157 new = f2fs_get_meta_page(sbi
, blk_addr
);
4159 return PTR_ERR(new);
4160 sum
= (struct f2fs_summary_block
*)page_address(new);
4162 if (IS_NODESEG(type
)) {
4163 if (__exist_node_summaries(sbi
)) {
4164 struct f2fs_summary
*ns
= &sum
->entries
[0];
4167 for (i
= 0; i
< BLKS_PER_SEG(sbi
); i
++, ns
++) {
4169 ns
->ofs_in_node
= 0;
4172 err
= f2fs_restore_node_summary(sbi
, segno
, sum
);
4178 /* set uncompleted segment to curseg */
4179 curseg
= CURSEG_I(sbi
, type
);
4180 mutex_lock(&curseg
->curseg_mutex
);
4182 /* update journal info */
4183 down_write(&curseg
->journal_rwsem
);
4184 memcpy(curseg
->journal
, &sum
->journal
, SUM_JOURNAL_SIZE
);
4185 up_write(&curseg
->journal_rwsem
);
4187 memcpy(curseg
->sum_blk
->entries
, sum
->entries
, SUM_ENTRY_SIZE
);
4188 memcpy(&curseg
->sum_blk
->footer
, &sum
->footer
, SUM_FOOTER_SIZE
);
4189 curseg
->next_segno
= segno
;
4190 reset_curseg(sbi
, type
, 0);
4191 curseg
->alloc_type
= ckpt
->alloc_type
[type
];
4192 curseg
->next_blkoff
= blk_off
;
4193 mutex_unlock(&curseg
->curseg_mutex
);
4195 f2fs_put_page(new, 1);
4199 static int restore_curseg_summaries(struct f2fs_sb_info
*sbi
)
4201 struct f2fs_journal
*sit_j
= CURSEG_I(sbi
, CURSEG_COLD_DATA
)->journal
;
4202 struct f2fs_journal
*nat_j
= CURSEG_I(sbi
, CURSEG_HOT_DATA
)->journal
;
4203 int type
= CURSEG_HOT_DATA
;
4206 if (is_set_ckpt_flags(sbi
, CP_COMPACT_SUM_FLAG
)) {
4207 int npages
= f2fs_npages_for_summary_flush(sbi
, true);
4210 f2fs_ra_meta_pages(sbi
, start_sum_block(sbi
), npages
,
4213 /* restore for compacted data summary */
4214 err
= read_compacted_summaries(sbi
);
4217 type
= CURSEG_HOT_NODE
;
4220 if (__exist_node_summaries(sbi
))
4221 f2fs_ra_meta_pages(sbi
,
4222 sum_blk_addr(sbi
, NR_CURSEG_PERSIST_TYPE
, type
),
4223 NR_CURSEG_PERSIST_TYPE
- type
, META_CP
, true);
4225 for (; type
<= CURSEG_COLD_NODE
; type
++) {
4226 err
= read_normal_summaries(sbi
, type
);
4231 /* sanity check for summary blocks */
4232 if (nats_in_cursum(nat_j
) > NAT_JOURNAL_ENTRIES
||
4233 sits_in_cursum(sit_j
) > SIT_JOURNAL_ENTRIES
) {
4234 f2fs_err(sbi
, "invalid journal entries nats %u sits %u",
4235 nats_in_cursum(nat_j
), sits_in_cursum(sit_j
));
4242 static void write_compacted_summaries(struct f2fs_sb_info
*sbi
, block_t blkaddr
)
4245 unsigned char *kaddr
;
4246 struct f2fs_summary
*summary
;
4247 struct curseg_info
*seg_i
;
4248 int written_size
= 0;
4251 page
= f2fs_grab_meta_page(sbi
, blkaddr
++);
4252 kaddr
= (unsigned char *)page_address(page
);
4253 memset(kaddr
, 0, PAGE_SIZE
);
4255 /* Step 1: write nat cache */
4256 seg_i
= CURSEG_I(sbi
, CURSEG_HOT_DATA
);
4257 memcpy(kaddr
, seg_i
->journal
, SUM_JOURNAL_SIZE
);
4258 written_size
+= SUM_JOURNAL_SIZE
;
4260 /* Step 2: write sit cache */
4261 seg_i
= CURSEG_I(sbi
, CURSEG_COLD_DATA
);
4262 memcpy(kaddr
+ written_size
, seg_i
->journal
, SUM_JOURNAL_SIZE
);
4263 written_size
+= SUM_JOURNAL_SIZE
;
4265 /* Step 3: write summary entries */
4266 for (i
= CURSEG_HOT_DATA
; i
<= CURSEG_COLD_DATA
; i
++) {
4267 seg_i
= CURSEG_I(sbi
, i
);
4268 for (j
= 0; j
< f2fs_curseg_valid_blocks(sbi
, i
); j
++) {
4270 page
= f2fs_grab_meta_page(sbi
, blkaddr
++);
4271 kaddr
= (unsigned char *)page_address(page
);
4272 memset(kaddr
, 0, PAGE_SIZE
);
4275 summary
= (struct f2fs_summary
*)(kaddr
+ written_size
);
4276 *summary
= seg_i
->sum_blk
->entries
[j
];
4277 written_size
+= SUMMARY_SIZE
;
4279 if (written_size
+ SUMMARY_SIZE
<= PAGE_SIZE
-
4283 set_page_dirty(page
);
4284 f2fs_put_page(page
, 1);
4289 set_page_dirty(page
);
4290 f2fs_put_page(page
, 1);
4294 static void write_normal_summaries(struct f2fs_sb_info
*sbi
,
4295 block_t blkaddr
, int type
)
4299 if (IS_DATASEG(type
))
4300 end
= type
+ NR_CURSEG_DATA_TYPE
;
4302 end
= type
+ NR_CURSEG_NODE_TYPE
;
4304 for (i
= type
; i
< end
; i
++)
4305 write_current_sum_page(sbi
, i
, blkaddr
+ (i
- type
));
4308 void f2fs_write_data_summaries(struct f2fs_sb_info
*sbi
, block_t start_blk
)
4310 if (is_set_ckpt_flags(sbi
, CP_COMPACT_SUM_FLAG
))
4311 write_compacted_summaries(sbi
, start_blk
);
4313 write_normal_summaries(sbi
, start_blk
, CURSEG_HOT_DATA
);
4316 void f2fs_write_node_summaries(struct f2fs_sb_info
*sbi
, block_t start_blk
)
4318 write_normal_summaries(sbi
, start_blk
, CURSEG_HOT_NODE
);
4321 int f2fs_lookup_journal_in_cursum(struct f2fs_journal
*journal
, int type
,
4322 unsigned int val
, int alloc
)
4326 if (type
== NAT_JOURNAL
) {
4327 for (i
= 0; i
< nats_in_cursum(journal
); i
++) {
4328 if (le32_to_cpu(nid_in_journal(journal
, i
)) == val
)
4331 if (alloc
&& __has_cursum_space(journal
, 1, NAT_JOURNAL
))
4332 return update_nats_in_cursum(journal
, 1);
4333 } else if (type
== SIT_JOURNAL
) {
4334 for (i
= 0; i
< sits_in_cursum(journal
); i
++)
4335 if (le32_to_cpu(segno_in_journal(journal
, i
)) == val
)
4337 if (alloc
&& __has_cursum_space(journal
, 1, SIT_JOURNAL
))
4338 return update_sits_in_cursum(journal
, 1);
4343 static struct page
*get_current_sit_page(struct f2fs_sb_info
*sbi
,
4346 return f2fs_get_meta_page(sbi
, current_sit_addr(sbi
, segno
));
4349 static struct page
*get_next_sit_page(struct f2fs_sb_info
*sbi
,
4352 struct sit_info
*sit_i
= SIT_I(sbi
);
4354 pgoff_t src_off
, dst_off
;
4356 src_off
= current_sit_addr(sbi
, start
);
4357 dst_off
= next_sit_addr(sbi
, src_off
);
4359 page
= f2fs_grab_meta_page(sbi
, dst_off
);
4360 seg_info_to_sit_page(sbi
, page
, start
);
4362 set_page_dirty(page
);
4363 set_to_next_sit(sit_i
, start
);
4368 static struct sit_entry_set
*grab_sit_entry_set(void)
4370 struct sit_entry_set
*ses
=
4371 f2fs_kmem_cache_alloc(sit_entry_set_slab
,
4372 GFP_NOFS
, true, NULL
);
4375 INIT_LIST_HEAD(&ses
->set_list
);
4379 static void release_sit_entry_set(struct sit_entry_set
*ses
)
4381 list_del(&ses
->set_list
);
4382 kmem_cache_free(sit_entry_set_slab
, ses
);
4385 static void adjust_sit_entry_set(struct sit_entry_set
*ses
,
4386 struct list_head
*head
)
4388 struct sit_entry_set
*next
= ses
;
4390 if (list_is_last(&ses
->set_list
, head
))
4393 list_for_each_entry_continue(next
, head
, set_list
)
4394 if (ses
->entry_cnt
<= next
->entry_cnt
) {
4395 list_move_tail(&ses
->set_list
, &next
->set_list
);
4399 list_move_tail(&ses
->set_list
, head
);
4402 static void add_sit_entry(unsigned int segno
, struct list_head
*head
)
4404 struct sit_entry_set
*ses
;
4405 unsigned int start_segno
= START_SEGNO(segno
);
4407 list_for_each_entry(ses
, head
, set_list
) {
4408 if (ses
->start_segno
== start_segno
) {
4410 adjust_sit_entry_set(ses
, head
);
4415 ses
= grab_sit_entry_set();
4417 ses
->start_segno
= start_segno
;
4419 list_add(&ses
->set_list
, head
);
4422 static void add_sits_in_set(struct f2fs_sb_info
*sbi
)
4424 struct f2fs_sm_info
*sm_info
= SM_I(sbi
);
4425 struct list_head
*set_list
= &sm_info
->sit_entry_set
;
4426 unsigned long *bitmap
= SIT_I(sbi
)->dirty_sentries_bitmap
;
4429 for_each_set_bit(segno
, bitmap
, MAIN_SEGS(sbi
))
4430 add_sit_entry(segno
, set_list
);
4433 static void remove_sits_in_journal(struct f2fs_sb_info
*sbi
)
4435 struct curseg_info
*curseg
= CURSEG_I(sbi
, CURSEG_COLD_DATA
);
4436 struct f2fs_journal
*journal
= curseg
->journal
;
4439 down_write(&curseg
->journal_rwsem
);
4440 for (i
= 0; i
< sits_in_cursum(journal
); i
++) {
4444 segno
= le32_to_cpu(segno_in_journal(journal
, i
));
4445 dirtied
= __mark_sit_entry_dirty(sbi
, segno
);
4448 add_sit_entry(segno
, &SM_I(sbi
)->sit_entry_set
);
4450 update_sits_in_cursum(journal
, -i
);
4451 up_write(&curseg
->journal_rwsem
);
4455 * CP calls this function, which flushes SIT entries including sit_journal,
4456 * and moves prefree segs to free segs.
4458 void f2fs_flush_sit_entries(struct f2fs_sb_info
*sbi
, struct cp_control
*cpc
)
4460 struct sit_info
*sit_i
= SIT_I(sbi
);
4461 unsigned long *bitmap
= sit_i
->dirty_sentries_bitmap
;
4462 struct curseg_info
*curseg
= CURSEG_I(sbi
, CURSEG_COLD_DATA
);
4463 struct f2fs_journal
*journal
= curseg
->journal
;
4464 struct sit_entry_set
*ses
, *tmp
;
4465 struct list_head
*head
= &SM_I(sbi
)->sit_entry_set
;
4466 bool to_journal
= !is_sbi_flag_set(sbi
, SBI_IS_RESIZEFS
);
4467 struct seg_entry
*se
;
4469 down_write(&sit_i
->sentry_lock
);
4471 if (!sit_i
->dirty_sentries
)
4475 * add and account sit entries of dirty bitmap in sit entry
4478 add_sits_in_set(sbi
);
4481 * if there are no enough space in journal to store dirty sit
4482 * entries, remove all entries from journal and add and account
4483 * them in sit entry set.
4485 if (!__has_cursum_space(journal
, sit_i
->dirty_sentries
, SIT_JOURNAL
) ||
4487 remove_sits_in_journal(sbi
);
4490 * there are two steps to flush sit entries:
4491 * #1, flush sit entries to journal in current cold data summary block.
4492 * #2, flush sit entries to sit page.
4494 list_for_each_entry_safe(ses
, tmp
, head
, set_list
) {
4495 struct page
*page
= NULL
;
4496 struct f2fs_sit_block
*raw_sit
= NULL
;
4497 unsigned int start_segno
= ses
->start_segno
;
4498 unsigned int end
= min(start_segno
+ SIT_ENTRY_PER_BLOCK
,
4499 (unsigned long)MAIN_SEGS(sbi
));
4500 unsigned int segno
= start_segno
;
4503 !__has_cursum_space(journal
, ses
->entry_cnt
, SIT_JOURNAL
))
4507 down_write(&curseg
->journal_rwsem
);
4509 page
= get_next_sit_page(sbi
, start_segno
);
4510 raw_sit
= page_address(page
);
4513 /* flush dirty sit entries in region of current sit set */
4514 for_each_set_bit_from(segno
, bitmap
, end
) {
4515 int offset
, sit_offset
;
4517 se
= get_seg_entry(sbi
, segno
);
4518 #ifdef CONFIG_F2FS_CHECK_FS
4519 if (memcmp(se
->cur_valid_map
, se
->cur_valid_map_mir
,
4520 SIT_VBLOCK_MAP_SIZE
))
4521 f2fs_bug_on(sbi
, 1);
4524 /* add discard candidates */
4525 if (!(cpc
->reason
& CP_DISCARD
)) {
4526 cpc
->trim_start
= segno
;
4527 add_discard_addrs(sbi
, cpc
, false);
4531 offset
= f2fs_lookup_journal_in_cursum(journal
,
4532 SIT_JOURNAL
, segno
, 1);
4533 f2fs_bug_on(sbi
, offset
< 0);
4534 segno_in_journal(journal
, offset
) =
4536 seg_info_to_raw_sit(se
,
4537 &sit_in_journal(journal
, offset
));
4538 check_block_count(sbi
, segno
,
4539 &sit_in_journal(journal
, offset
));
4541 sit_offset
= SIT_ENTRY_OFFSET(sit_i
, segno
);
4542 seg_info_to_raw_sit(se
,
4543 &raw_sit
->entries
[sit_offset
]);
4544 check_block_count(sbi
, segno
,
4545 &raw_sit
->entries
[sit_offset
]);
4548 __clear_bit(segno
, bitmap
);
4549 sit_i
->dirty_sentries
--;
4554 up_write(&curseg
->journal_rwsem
);
4556 f2fs_put_page(page
, 1);
4558 f2fs_bug_on(sbi
, ses
->entry_cnt
);
4559 release_sit_entry_set(ses
);
4562 f2fs_bug_on(sbi
, !list_empty(head
));
4563 f2fs_bug_on(sbi
, sit_i
->dirty_sentries
);
4565 if (cpc
->reason
& CP_DISCARD
) {
4566 __u64 trim_start
= cpc
->trim_start
;
4568 for (; cpc
->trim_start
<= cpc
->trim_end
; cpc
->trim_start
++)
4569 add_discard_addrs(sbi
, cpc
, false);
4571 cpc
->trim_start
= trim_start
;
4573 up_write(&sit_i
->sentry_lock
);
4575 set_prefree_as_free_segments(sbi
);
4578 static int build_sit_info(struct f2fs_sb_info
*sbi
)
4580 struct f2fs_super_block
*raw_super
= F2FS_RAW_SUPER(sbi
);
4581 struct sit_info
*sit_i
;
4582 unsigned int sit_segs
, start
;
4583 char *src_bitmap
, *bitmap
;
4584 unsigned int bitmap_size
, main_bitmap_size
, sit_bitmap_size
;
4585 unsigned int discard_map
= f2fs_block_unit_discard(sbi
) ? 1 : 0;
4587 /* allocate memory for SIT information */
4588 sit_i
= f2fs_kzalloc(sbi
, sizeof(struct sit_info
), GFP_KERNEL
);
4592 SM_I(sbi
)->sit_info
= sit_i
;
4595 f2fs_kvzalloc(sbi
, array_size(sizeof(struct seg_entry
),
4598 if (!sit_i
->sentries
)
4601 main_bitmap_size
= f2fs_bitmap_size(MAIN_SEGS(sbi
));
4602 sit_i
->dirty_sentries_bitmap
= f2fs_kvzalloc(sbi
, main_bitmap_size
,
4604 if (!sit_i
->dirty_sentries_bitmap
)
4607 #ifdef CONFIG_F2FS_CHECK_FS
4608 bitmap_size
= MAIN_SEGS(sbi
) * SIT_VBLOCK_MAP_SIZE
* (3 + discard_map
);
4610 bitmap_size
= MAIN_SEGS(sbi
) * SIT_VBLOCK_MAP_SIZE
* (2 + discard_map
);
4612 sit_i
->bitmap
= f2fs_kvzalloc(sbi
, bitmap_size
, GFP_KERNEL
);
4616 bitmap
= sit_i
->bitmap
;
4618 for (start
= 0; start
< MAIN_SEGS(sbi
); start
++) {
4619 sit_i
->sentries
[start
].cur_valid_map
= bitmap
;
4620 bitmap
+= SIT_VBLOCK_MAP_SIZE
;
4622 sit_i
->sentries
[start
].ckpt_valid_map
= bitmap
;
4623 bitmap
+= SIT_VBLOCK_MAP_SIZE
;
4625 #ifdef CONFIG_F2FS_CHECK_FS
4626 sit_i
->sentries
[start
].cur_valid_map_mir
= bitmap
;
4627 bitmap
+= SIT_VBLOCK_MAP_SIZE
;
4631 sit_i
->sentries
[start
].discard_map
= bitmap
;
4632 bitmap
+= SIT_VBLOCK_MAP_SIZE
;
4636 sit_i
->tmp_map
= f2fs_kzalloc(sbi
, SIT_VBLOCK_MAP_SIZE
, GFP_KERNEL
);
4637 if (!sit_i
->tmp_map
)
4640 if (__is_large_section(sbi
)) {
4641 sit_i
->sec_entries
=
4642 f2fs_kvzalloc(sbi
, array_size(sizeof(struct sec_entry
),
4645 if (!sit_i
->sec_entries
)
4649 /* get information related with SIT */
4650 sit_segs
= le32_to_cpu(raw_super
->segment_count_sit
) >> 1;
4652 /* setup SIT bitmap from ckeckpoint pack */
4653 sit_bitmap_size
= __bitmap_size(sbi
, SIT_BITMAP
);
4654 src_bitmap
= __bitmap_ptr(sbi
, SIT_BITMAP
);
4656 sit_i
->sit_bitmap
= kmemdup(src_bitmap
, sit_bitmap_size
, GFP_KERNEL
);
4657 if (!sit_i
->sit_bitmap
)
4660 #ifdef CONFIG_F2FS_CHECK_FS
4661 sit_i
->sit_bitmap_mir
= kmemdup(src_bitmap
,
4662 sit_bitmap_size
, GFP_KERNEL
);
4663 if (!sit_i
->sit_bitmap_mir
)
4666 sit_i
->invalid_segmap
= f2fs_kvzalloc(sbi
,
4667 main_bitmap_size
, GFP_KERNEL
);
4668 if (!sit_i
->invalid_segmap
)
4672 sit_i
->sit_base_addr
= le32_to_cpu(raw_super
->sit_blkaddr
);
4673 sit_i
->sit_blocks
= SEGS_TO_BLKS(sbi
, sit_segs
);
4674 sit_i
->written_valid_blocks
= 0;
4675 sit_i
->bitmap_size
= sit_bitmap_size
;
4676 sit_i
->dirty_sentries
= 0;
4677 sit_i
->sents_per_block
= SIT_ENTRY_PER_BLOCK
;
4678 sit_i
->elapsed_time
= le64_to_cpu(sbi
->ckpt
->elapsed_time
);
4679 sit_i
->mounted_time
= ktime_get_boottime_seconds();
4680 init_rwsem(&sit_i
->sentry_lock
);
4684 static int build_free_segmap(struct f2fs_sb_info
*sbi
)
4686 struct free_segmap_info
*free_i
;
4687 unsigned int bitmap_size
, sec_bitmap_size
;
4689 /* allocate memory for free segmap information */
4690 free_i
= f2fs_kzalloc(sbi
, sizeof(struct free_segmap_info
), GFP_KERNEL
);
4694 SM_I(sbi
)->free_info
= free_i
;
4696 bitmap_size
= f2fs_bitmap_size(MAIN_SEGS(sbi
));
4697 free_i
->free_segmap
= f2fs_kvmalloc(sbi
, bitmap_size
, GFP_KERNEL
);
4698 if (!free_i
->free_segmap
)
4701 sec_bitmap_size
= f2fs_bitmap_size(MAIN_SECS(sbi
));
4702 free_i
->free_secmap
= f2fs_kvmalloc(sbi
, sec_bitmap_size
, GFP_KERNEL
);
4703 if (!free_i
->free_secmap
)
4706 /* set all segments as dirty temporarily */
4707 memset(free_i
->free_segmap
, 0xff, bitmap_size
);
4708 memset(free_i
->free_secmap
, 0xff, sec_bitmap_size
);
4710 /* init free segmap information */
4711 free_i
->start_segno
= GET_SEGNO_FROM_SEG0(sbi
, MAIN_BLKADDR(sbi
));
4712 free_i
->free_segments
= 0;
4713 free_i
->free_sections
= 0;
4714 spin_lock_init(&free_i
->segmap_lock
);
4718 static int build_curseg(struct f2fs_sb_info
*sbi
)
4720 struct curseg_info
*array
;
4723 array
= f2fs_kzalloc(sbi
, array_size(NR_CURSEG_TYPE
,
4724 sizeof(*array
)), GFP_KERNEL
);
4728 SM_I(sbi
)->curseg_array
= array
;
4730 for (i
= 0; i
< NO_CHECK_TYPE
; i
++) {
4731 mutex_init(&array
[i
].curseg_mutex
);
4732 array
[i
].sum_blk
= f2fs_kzalloc(sbi
, PAGE_SIZE
, GFP_KERNEL
);
4733 if (!array
[i
].sum_blk
)
4735 init_rwsem(&array
[i
].journal_rwsem
);
4736 array
[i
].journal
= f2fs_kzalloc(sbi
,
4737 sizeof(struct f2fs_journal
), GFP_KERNEL
);
4738 if (!array
[i
].journal
)
4740 if (i
< NR_PERSISTENT_LOG
)
4741 array
[i
].seg_type
= CURSEG_HOT_DATA
+ i
;
4742 else if (i
== CURSEG_COLD_DATA_PINNED
)
4743 array
[i
].seg_type
= CURSEG_COLD_DATA
;
4744 else if (i
== CURSEG_ALL_DATA_ATGC
)
4745 array
[i
].seg_type
= CURSEG_COLD_DATA
;
4746 reset_curseg_fields(&array
[i
]);
4748 return restore_curseg_summaries(sbi
);
4751 static int build_sit_entries(struct f2fs_sb_info
*sbi
)
4753 struct sit_info
*sit_i
= SIT_I(sbi
);
4754 struct curseg_info
*curseg
= CURSEG_I(sbi
, CURSEG_COLD_DATA
);
4755 struct f2fs_journal
*journal
= curseg
->journal
;
4756 struct seg_entry
*se
;
4757 struct f2fs_sit_entry sit
;
4758 int sit_blk_cnt
= SIT_BLK_CNT(sbi
);
4759 unsigned int i
, start
, end
;
4760 unsigned int readed
, start_blk
= 0;
4762 block_t sit_valid_blocks
[2] = {0, 0};
4765 readed
= f2fs_ra_meta_pages(sbi
, start_blk
, BIO_MAX_VECS
,
4768 start
= start_blk
* sit_i
->sents_per_block
;
4769 end
= (start_blk
+ readed
) * sit_i
->sents_per_block
;
4771 for (; start
< end
&& start
< MAIN_SEGS(sbi
); start
++) {
4772 struct f2fs_sit_block
*sit_blk
;
4775 se
= &sit_i
->sentries
[start
];
4776 page
= get_current_sit_page(sbi
, start
);
4778 return PTR_ERR(page
);
4779 sit_blk
= (struct f2fs_sit_block
*)page_address(page
);
4780 sit
= sit_blk
->entries
[SIT_ENTRY_OFFSET(sit_i
, start
)];
4781 f2fs_put_page(page
, 1);
4783 err
= check_block_count(sbi
, start
, &sit
);
4786 seg_info_from_raw_sit(se
, &sit
);
4788 if (se
->type
>= NR_PERSISTENT_LOG
) {
4789 f2fs_err(sbi
, "Invalid segment type: %u, segno: %u",
4791 f2fs_handle_error(sbi
,
4792 ERROR_INCONSISTENT_SUM_TYPE
);
4793 return -EFSCORRUPTED
;
4796 sit_valid_blocks
[SE_PAGETYPE(se
)] += se
->valid_blocks
;
4798 if (!f2fs_block_unit_discard(sbi
))
4799 goto init_discard_map_done
;
4801 /* build discard map only one time */
4802 if (is_set_ckpt_flags(sbi
, CP_TRIMMED_FLAG
)) {
4803 memset(se
->discard_map
, 0xff,
4804 SIT_VBLOCK_MAP_SIZE
);
4805 goto init_discard_map_done
;
4807 memcpy(se
->discard_map
, se
->cur_valid_map
,
4808 SIT_VBLOCK_MAP_SIZE
);
4809 sbi
->discard_blks
+= BLKS_PER_SEG(sbi
) -
4811 init_discard_map_done
:
4812 if (__is_large_section(sbi
))
4813 get_sec_entry(sbi
, start
)->valid_blocks
+=
4816 start_blk
+= readed
;
4817 } while (start_blk
< sit_blk_cnt
);
4819 down_read(&curseg
->journal_rwsem
);
4820 for (i
= 0; i
< sits_in_cursum(journal
); i
++) {
4821 unsigned int old_valid_blocks
;
4823 start
= le32_to_cpu(segno_in_journal(journal
, i
));
4824 if (start
>= MAIN_SEGS(sbi
)) {
4825 f2fs_err(sbi
, "Wrong journal entry on segno %u",
4827 err
= -EFSCORRUPTED
;
4828 f2fs_handle_error(sbi
, ERROR_CORRUPTED_JOURNAL
);
4832 se
= &sit_i
->sentries
[start
];
4833 sit
= sit_in_journal(journal
, i
);
4835 old_valid_blocks
= se
->valid_blocks
;
4837 sit_valid_blocks
[SE_PAGETYPE(se
)] -= old_valid_blocks
;
4839 err
= check_block_count(sbi
, start
, &sit
);
4842 seg_info_from_raw_sit(se
, &sit
);
4844 if (se
->type
>= NR_PERSISTENT_LOG
) {
4845 f2fs_err(sbi
, "Invalid segment type: %u, segno: %u",
4847 err
= -EFSCORRUPTED
;
4848 f2fs_handle_error(sbi
, ERROR_INCONSISTENT_SUM_TYPE
);
4852 sit_valid_blocks
[SE_PAGETYPE(se
)] += se
->valid_blocks
;
4854 if (f2fs_block_unit_discard(sbi
)) {
4855 if (is_set_ckpt_flags(sbi
, CP_TRIMMED_FLAG
)) {
4856 memset(se
->discard_map
, 0xff, SIT_VBLOCK_MAP_SIZE
);
4858 memcpy(se
->discard_map
, se
->cur_valid_map
,
4859 SIT_VBLOCK_MAP_SIZE
);
4860 sbi
->discard_blks
+= old_valid_blocks
;
4861 sbi
->discard_blks
-= se
->valid_blocks
;
4865 if (__is_large_section(sbi
)) {
4866 get_sec_entry(sbi
, start
)->valid_blocks
+=
4868 get_sec_entry(sbi
, start
)->valid_blocks
-=
4872 up_read(&curseg
->journal_rwsem
);
4877 if (sit_valid_blocks
[NODE
] != valid_node_count(sbi
)) {
4878 f2fs_err(sbi
, "SIT is corrupted node# %u vs %u",
4879 sit_valid_blocks
[NODE
], valid_node_count(sbi
));
4880 f2fs_handle_error(sbi
, ERROR_INCONSISTENT_NODE_COUNT
);
4881 return -EFSCORRUPTED
;
4884 if (sit_valid_blocks
[DATA
] + sit_valid_blocks
[NODE
] >
4885 valid_user_blocks(sbi
)) {
4886 f2fs_err(sbi
, "SIT is corrupted data# %u %u vs %u",
4887 sit_valid_blocks
[DATA
], sit_valid_blocks
[NODE
],
4888 valid_user_blocks(sbi
));
4889 f2fs_handle_error(sbi
, ERROR_INCONSISTENT_BLOCK_COUNT
);
4890 return -EFSCORRUPTED
;
4896 static void init_free_segmap(struct f2fs_sb_info
*sbi
)
4900 struct seg_entry
*sentry
;
4902 for (start
= 0; start
< MAIN_SEGS(sbi
); start
++) {
4903 if (f2fs_usable_blks_in_seg(sbi
, start
) == 0)
4905 sentry
= get_seg_entry(sbi
, start
);
4906 if (!sentry
->valid_blocks
)
4907 __set_free(sbi
, start
);
4909 SIT_I(sbi
)->written_valid_blocks
+=
4910 sentry
->valid_blocks
;
4913 /* set use the current segments */
4914 for (type
= CURSEG_HOT_DATA
; type
<= CURSEG_COLD_NODE
; type
++) {
4915 struct curseg_info
*curseg_t
= CURSEG_I(sbi
, type
);
4917 __set_test_and_inuse(sbi
, curseg_t
->segno
);
4921 static void init_dirty_segmap(struct f2fs_sb_info
*sbi
)
4923 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
4924 struct free_segmap_info
*free_i
= FREE_I(sbi
);
4925 unsigned int segno
= 0, offset
= 0, secno
;
4926 block_t valid_blocks
, usable_blks_in_seg
;
4929 /* find dirty segment based on free segmap */
4930 segno
= find_next_inuse(free_i
, MAIN_SEGS(sbi
), offset
);
4931 if (segno
>= MAIN_SEGS(sbi
))
4934 valid_blocks
= get_valid_blocks(sbi
, segno
, false);
4935 usable_blks_in_seg
= f2fs_usable_blks_in_seg(sbi
, segno
);
4936 if (valid_blocks
== usable_blks_in_seg
|| !valid_blocks
)
4938 if (valid_blocks
> usable_blks_in_seg
) {
4939 f2fs_bug_on(sbi
, 1);
4942 mutex_lock(&dirty_i
->seglist_lock
);
4943 __locate_dirty_segment(sbi
, segno
, DIRTY
);
4944 mutex_unlock(&dirty_i
->seglist_lock
);
4947 if (!__is_large_section(sbi
))
4950 mutex_lock(&dirty_i
->seglist_lock
);
4951 for (segno
= 0; segno
< MAIN_SEGS(sbi
); segno
+= SEGS_PER_SEC(sbi
)) {
4952 valid_blocks
= get_valid_blocks(sbi
, segno
, true);
4953 secno
= GET_SEC_FROM_SEG(sbi
, segno
);
4955 if (!valid_blocks
|| valid_blocks
== CAP_BLKS_PER_SEC(sbi
))
4957 if (IS_CURSEC(sbi
, secno
))
4959 set_bit(secno
, dirty_i
->dirty_secmap
);
4961 mutex_unlock(&dirty_i
->seglist_lock
);
4964 static int init_victim_secmap(struct f2fs_sb_info
*sbi
)
4966 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
4967 unsigned int bitmap_size
= f2fs_bitmap_size(MAIN_SECS(sbi
));
4969 dirty_i
->victim_secmap
= f2fs_kvzalloc(sbi
, bitmap_size
, GFP_KERNEL
);
4970 if (!dirty_i
->victim_secmap
)
4973 dirty_i
->pinned_secmap
= f2fs_kvzalloc(sbi
, bitmap_size
, GFP_KERNEL
);
4974 if (!dirty_i
->pinned_secmap
)
4977 dirty_i
->pinned_secmap_cnt
= 0;
4978 dirty_i
->enable_pin_section
= true;
4982 static int build_dirty_segmap(struct f2fs_sb_info
*sbi
)
4984 struct dirty_seglist_info
*dirty_i
;
4985 unsigned int bitmap_size
, i
;
4987 /* allocate memory for dirty segments list information */
4988 dirty_i
= f2fs_kzalloc(sbi
, sizeof(struct dirty_seglist_info
),
4993 SM_I(sbi
)->dirty_info
= dirty_i
;
4994 mutex_init(&dirty_i
->seglist_lock
);
4996 bitmap_size
= f2fs_bitmap_size(MAIN_SEGS(sbi
));
4998 for (i
= 0; i
< NR_DIRTY_TYPE
; i
++) {
4999 dirty_i
->dirty_segmap
[i
] = f2fs_kvzalloc(sbi
, bitmap_size
,
5001 if (!dirty_i
->dirty_segmap
[i
])
5005 if (__is_large_section(sbi
)) {
5006 bitmap_size
= f2fs_bitmap_size(MAIN_SECS(sbi
));
5007 dirty_i
->dirty_secmap
= f2fs_kvzalloc(sbi
,
5008 bitmap_size
, GFP_KERNEL
);
5009 if (!dirty_i
->dirty_secmap
)
5013 init_dirty_segmap(sbi
);
5014 return init_victim_secmap(sbi
);
5017 static int sanity_check_curseg(struct f2fs_sb_info
*sbi
)
5022 * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr;
5023 * In LFS curseg, all blkaddr after .next_blkoff should be unused.
5025 for (i
= 0; i
< NR_PERSISTENT_LOG
; i
++) {
5026 struct curseg_info
*curseg
= CURSEG_I(sbi
, i
);
5027 struct seg_entry
*se
= get_seg_entry(sbi
, curseg
->segno
);
5028 unsigned int blkofs
= curseg
->next_blkoff
;
5030 if (f2fs_sb_has_readonly(sbi
) &&
5031 i
!= CURSEG_HOT_DATA
&& i
!= CURSEG_HOT_NODE
)
5034 sanity_check_seg_type(sbi
, curseg
->seg_type
);
5036 if (curseg
->alloc_type
!= LFS
&& curseg
->alloc_type
!= SSR
) {
5038 "Current segment has invalid alloc_type:%d",
5039 curseg
->alloc_type
);
5040 f2fs_handle_error(sbi
, ERROR_INVALID_CURSEG
);
5041 return -EFSCORRUPTED
;
5044 if (f2fs_test_bit(blkofs
, se
->cur_valid_map
))
5047 if (curseg
->alloc_type
== SSR
)
5050 for (blkofs
+= 1; blkofs
< BLKS_PER_SEG(sbi
); blkofs
++) {
5051 if (!f2fs_test_bit(blkofs
, se
->cur_valid_map
))
5055 "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
5056 i
, curseg
->segno
, curseg
->alloc_type
,
5057 curseg
->next_blkoff
, blkofs
);
5058 f2fs_handle_error(sbi
, ERROR_INVALID_CURSEG
);
5059 return -EFSCORRUPTED
;
5065 #ifdef CONFIG_BLK_DEV_ZONED
5066 static int check_zone_write_pointer(struct f2fs_sb_info
*sbi
,
5067 struct f2fs_dev_info
*fdev
,
5068 struct blk_zone
*zone
)
5070 unsigned int zone_segno
;
5071 block_t zone_block
, valid_block_cnt
;
5072 unsigned int log_sectors_per_block
= sbi
->log_blocksize
- SECTOR_SHIFT
;
5074 unsigned int nofs_flags
;
5076 if (zone
->type
!= BLK_ZONE_TYPE_SEQWRITE_REQ
)
5079 zone_block
= fdev
->start_blk
+ (zone
->start
>> log_sectors_per_block
);
5080 zone_segno
= GET_SEGNO(sbi
, zone_block
);
5083 * Skip check of zones cursegs point to, since
5084 * fix_curseg_write_pointer() checks them.
5086 if (zone_segno
>= MAIN_SEGS(sbi
))
5090 * Get # of valid block of the zone.
5092 valid_block_cnt
= get_valid_blocks(sbi
, zone_segno
, true);
5093 if (IS_CURSEC(sbi
, GET_SEC_FROM_SEG(sbi
, zone_segno
))) {
5094 f2fs_notice(sbi
, "Open zones: valid block[0x%x,0x%x] cond[%s]",
5095 zone_segno
, valid_block_cnt
,
5096 blk_zone_cond_str(zone
->cond
));
5100 if ((!valid_block_cnt
&& zone
->cond
== BLK_ZONE_COND_EMPTY
) ||
5101 (valid_block_cnt
&& zone
->cond
== BLK_ZONE_COND_FULL
))
5104 if (!valid_block_cnt
) {
5105 f2fs_notice(sbi
, "Zone without valid block has non-zero write "
5106 "pointer. Reset the write pointer: cond[%s]",
5107 blk_zone_cond_str(zone
->cond
));
5108 ret
= __f2fs_issue_discard_zone(sbi
, fdev
->bdev
, zone_block
,
5109 zone
->len
>> log_sectors_per_block
);
5111 f2fs_err(sbi
, "Discard zone failed: %s (errno=%d)",
5117 * If there are valid blocks and the write pointer doesn't match
5118 * with them, we need to report the inconsistency and fill
5119 * the zone till the end to close the zone. This inconsistency
5120 * does not cause write error because the zone will not be
5121 * selected for write operation until it get discarded.
5123 f2fs_notice(sbi
, "Valid blocks are not aligned with write "
5124 "pointer: valid block[0x%x,0x%x] cond[%s]",
5125 zone_segno
, valid_block_cnt
, blk_zone_cond_str(zone
->cond
));
5127 nofs_flags
= memalloc_nofs_save();
5128 ret
= blkdev_zone_mgmt(fdev
->bdev
, REQ_OP_ZONE_FINISH
,
5129 zone
->start
, zone
->len
);
5130 memalloc_nofs_restore(nofs_flags
);
5131 if (ret
== -EOPNOTSUPP
) {
5132 ret
= blkdev_issue_zeroout(fdev
->bdev
, zone
->wp
,
5133 zone
->len
- (zone
->wp
- zone
->start
),
5136 f2fs_err(sbi
, "Fill up zone failed: %s (errno=%d)",
5139 f2fs_err(sbi
, "Finishing zone failed: %s (errno=%d)",
5146 static struct f2fs_dev_info
*get_target_zoned_dev(struct f2fs_sb_info
*sbi
,
5147 block_t zone_blkaddr
)
5151 for (i
= 0; i
< sbi
->s_ndevs
; i
++) {
5152 if (!bdev_is_zoned(FDEV(i
).bdev
))
5154 if (sbi
->s_ndevs
== 1 || (FDEV(i
).start_blk
<= zone_blkaddr
&&
5155 zone_blkaddr
<= FDEV(i
).end_blk
))
5162 static int report_one_zone_cb(struct blk_zone
*zone
, unsigned int idx
,
5165 memcpy(data
, zone
, sizeof(struct blk_zone
));
5169 static int fix_curseg_write_pointer(struct f2fs_sb_info
*sbi
, int type
)
5171 struct curseg_info
*cs
= CURSEG_I(sbi
, type
);
5172 struct f2fs_dev_info
*zbd
;
5173 struct blk_zone zone
;
5174 unsigned int cs_section
, wp_segno
, wp_blkoff
, wp_sector_off
;
5175 block_t cs_zone_block
, wp_block
;
5176 unsigned int log_sectors_per_block
= sbi
->log_blocksize
- SECTOR_SHIFT
;
5177 sector_t zone_sector
;
5180 cs_section
= GET_SEC_FROM_SEG(sbi
, cs
->segno
);
5181 cs_zone_block
= START_BLOCK(sbi
, GET_SEG_FROM_SEC(sbi
, cs_section
));
5183 zbd
= get_target_zoned_dev(sbi
, cs_zone_block
);
5187 /* report zone for the sector the curseg points to */
5188 zone_sector
= (sector_t
)(cs_zone_block
- zbd
->start_blk
)
5189 << log_sectors_per_block
;
5190 err
= blkdev_report_zones(zbd
->bdev
, zone_sector
, 1,
5191 report_one_zone_cb
, &zone
);
5193 f2fs_err(sbi
, "Report zone failed: %s errno=(%d)",
5198 if (zone
.type
!= BLK_ZONE_TYPE_SEQWRITE_REQ
)
5202 * When safely unmounted in the previous mount, we could use current
5203 * segments. Otherwise, allocate new sections.
5205 if (is_set_ckpt_flags(sbi
, CP_UMOUNT_FLAG
)) {
5206 wp_block
= zbd
->start_blk
+ (zone
.wp
>> log_sectors_per_block
);
5207 wp_segno
= GET_SEGNO(sbi
, wp_block
);
5208 wp_blkoff
= wp_block
- START_BLOCK(sbi
, wp_segno
);
5209 wp_sector_off
= zone
.wp
& GENMASK(log_sectors_per_block
- 1, 0);
5211 if (cs
->segno
== wp_segno
&& cs
->next_blkoff
== wp_blkoff
&&
5215 f2fs_notice(sbi
, "Unaligned curseg[%d] with write pointer: "
5216 "curseg[0x%x,0x%x] wp[0x%x,0x%x]", type
, cs
->segno
,
5217 cs
->next_blkoff
, wp_segno
, wp_blkoff
);
5220 /* Allocate a new section if it's not new. */
5221 if (cs
->next_blkoff
||
5222 cs
->segno
!= GET_SEG_FROM_SEC(sbi
, GET_ZONE_FROM_SEC(sbi
, cs_section
))) {
5223 unsigned int old_segno
= cs
->segno
, old_blkoff
= cs
->next_blkoff
;
5225 f2fs_allocate_new_section(sbi
, type
, true);
5226 f2fs_notice(sbi
, "Assign new section to curseg[%d]: "
5227 "[0x%x,0x%x] -> [0x%x,0x%x]",
5228 type
, old_segno
, old_blkoff
,
5229 cs
->segno
, cs
->next_blkoff
);
5232 /* check consistency of the zone curseg pointed to */
5233 if (check_zone_write_pointer(sbi
, zbd
, &zone
))
5236 /* check newly assigned zone */
5237 cs_section
= GET_SEC_FROM_SEG(sbi
, cs
->segno
);
5238 cs_zone_block
= START_BLOCK(sbi
, GET_SEG_FROM_SEC(sbi
, cs_section
));
5240 zbd
= get_target_zoned_dev(sbi
, cs_zone_block
);
5244 zone_sector
= (sector_t
)(cs_zone_block
- zbd
->start_blk
)
5245 << log_sectors_per_block
;
5246 err
= blkdev_report_zones(zbd
->bdev
, zone_sector
, 1,
5247 report_one_zone_cb
, &zone
);
5249 f2fs_err(sbi
, "Report zone failed: %s errno=(%d)",
5254 if (zone
.type
!= BLK_ZONE_TYPE_SEQWRITE_REQ
)
5257 if (zone
.wp
!= zone
.start
) {
5259 "New zone for curseg[%d] is not yet discarded. "
5260 "Reset the zone: curseg[0x%x,0x%x]",
5261 type
, cs
->segno
, cs
->next_blkoff
);
5262 err
= __f2fs_issue_discard_zone(sbi
, zbd
->bdev
, cs_zone_block
,
5263 zone
.len
>> log_sectors_per_block
);
5265 f2fs_err(sbi
, "Discard zone failed: %s (errno=%d)",
5274 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info
*sbi
)
5278 for (i
= 0; i
< NR_PERSISTENT_LOG
; i
++) {
5279 ret
= fix_curseg_write_pointer(sbi
, i
);
5287 struct check_zone_write_pointer_args
{
5288 struct f2fs_sb_info
*sbi
;
5289 struct f2fs_dev_info
*fdev
;
5292 static int check_zone_write_pointer_cb(struct blk_zone
*zone
, unsigned int idx
,
5295 struct check_zone_write_pointer_args
*args
;
5297 args
= (struct check_zone_write_pointer_args
*)data
;
5299 return check_zone_write_pointer(args
->sbi
, args
->fdev
, zone
);
5302 int f2fs_check_write_pointer(struct f2fs_sb_info
*sbi
)
5305 struct check_zone_write_pointer_args args
;
5307 for (i
= 0; i
< sbi
->s_ndevs
; i
++) {
5308 if (!bdev_is_zoned(FDEV(i
).bdev
))
5312 args
.fdev
= &FDEV(i
);
5313 ret
= blkdev_report_zones(FDEV(i
).bdev
, 0, BLK_ALL_ZONES
,
5314 check_zone_write_pointer_cb
, &args
);
5323 * Return the number of usable blocks in a segment. The number of blocks
5324 * returned is always equal to the number of blocks in a segment for
5325 * segments fully contained within a sequential zone capacity or a
5326 * conventional zone. For segments partially contained in a sequential
5327 * zone capacity, the number of usable blocks up to the zone capacity
5328 * is returned. 0 is returned in all other cases.
5330 static inline unsigned int f2fs_usable_zone_blks_in_seg(
5331 struct f2fs_sb_info
*sbi
, unsigned int segno
)
5333 block_t seg_start
, sec_start_blkaddr
, sec_cap_blkaddr
;
5336 if (!sbi
->unusable_blocks_per_sec
)
5337 return BLKS_PER_SEG(sbi
);
5339 secno
= GET_SEC_FROM_SEG(sbi
, segno
);
5340 seg_start
= START_BLOCK(sbi
, segno
);
5341 sec_start_blkaddr
= START_BLOCK(sbi
, GET_SEG_FROM_SEC(sbi
, secno
));
5342 sec_cap_blkaddr
= sec_start_blkaddr
+ CAP_BLKS_PER_SEC(sbi
);
5345 * If segment starts before zone capacity and spans beyond
5346 * zone capacity, then usable blocks are from seg start to
5347 * zone capacity. If the segment starts after the zone capacity,
5348 * then there are no usable blocks.
5350 if (seg_start
>= sec_cap_blkaddr
)
5352 if (seg_start
+ BLKS_PER_SEG(sbi
) > sec_cap_blkaddr
)
5353 return sec_cap_blkaddr
- seg_start
;
5355 return BLKS_PER_SEG(sbi
);
5358 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info
*sbi
)
5363 int f2fs_check_write_pointer(struct f2fs_sb_info
*sbi
)
5368 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info
*sbi
,
5375 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info
*sbi
,
5378 if (f2fs_sb_has_blkzoned(sbi
))
5379 return f2fs_usable_zone_blks_in_seg(sbi
, segno
);
5381 return BLKS_PER_SEG(sbi
);
5384 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info
*sbi
,
5387 if (f2fs_sb_has_blkzoned(sbi
))
5388 return CAP_SEGS_PER_SEC(sbi
);
5390 return SEGS_PER_SEC(sbi
);
5394 * Update min, max modified time for cost-benefit GC algorithm
5396 static void init_min_max_mtime(struct f2fs_sb_info
*sbi
)
5398 struct sit_info
*sit_i
= SIT_I(sbi
);
5401 down_write(&sit_i
->sentry_lock
);
5403 sit_i
->min_mtime
= ULLONG_MAX
;
5405 for (segno
= 0; segno
< MAIN_SEGS(sbi
); segno
+= SEGS_PER_SEC(sbi
)) {
5407 unsigned long long mtime
= 0;
5409 for (i
= 0; i
< SEGS_PER_SEC(sbi
); i
++)
5410 mtime
+= get_seg_entry(sbi
, segno
+ i
)->mtime
;
5412 mtime
= div_u64(mtime
, SEGS_PER_SEC(sbi
));
5414 if (sit_i
->min_mtime
> mtime
)
5415 sit_i
->min_mtime
= mtime
;
5417 sit_i
->max_mtime
= get_mtime(sbi
, false);
5418 sit_i
->dirty_max_mtime
= 0;
5419 up_write(&sit_i
->sentry_lock
);
5422 int f2fs_build_segment_manager(struct f2fs_sb_info
*sbi
)
5424 struct f2fs_super_block
*raw_super
= F2FS_RAW_SUPER(sbi
);
5425 struct f2fs_checkpoint
*ckpt
= F2FS_CKPT(sbi
);
5426 struct f2fs_sm_info
*sm_info
;
5429 sm_info
= f2fs_kzalloc(sbi
, sizeof(struct f2fs_sm_info
), GFP_KERNEL
);
5434 sbi
->sm_info
= sm_info
;
5435 sm_info
->seg0_blkaddr
= le32_to_cpu(raw_super
->segment0_blkaddr
);
5436 sm_info
->main_blkaddr
= le32_to_cpu(raw_super
->main_blkaddr
);
5437 sm_info
->segment_count
= le32_to_cpu(raw_super
->segment_count
);
5438 sm_info
->reserved_segments
= le32_to_cpu(ckpt
->rsvd_segment_count
);
5439 sm_info
->ovp_segments
= le32_to_cpu(ckpt
->overprov_segment_count
);
5440 sm_info
->main_segments
= le32_to_cpu(raw_super
->segment_count_main
);
5441 sm_info
->ssa_blkaddr
= le32_to_cpu(raw_super
->ssa_blkaddr
);
5442 sm_info
->rec_prefree_segments
= sm_info
->main_segments
*
5443 DEF_RECLAIM_PREFREE_SEGMENTS
/ 100;
5444 if (sm_info
->rec_prefree_segments
> DEF_MAX_RECLAIM_PREFREE_SEGMENTS
)
5445 sm_info
->rec_prefree_segments
= DEF_MAX_RECLAIM_PREFREE_SEGMENTS
;
5447 if (!f2fs_lfs_mode(sbi
))
5448 sm_info
->ipu_policy
= BIT(F2FS_IPU_FSYNC
);
5449 sm_info
->min_ipu_util
= DEF_MIN_IPU_UTIL
;
5450 sm_info
->min_fsync_blocks
= DEF_MIN_FSYNC_BLOCKS
;
5451 sm_info
->min_seq_blocks
= BLKS_PER_SEG(sbi
);
5452 sm_info
->min_hot_blocks
= DEF_MIN_HOT_BLOCKS
;
5453 sm_info
->min_ssr_sections
= reserved_sections(sbi
);
5455 INIT_LIST_HEAD(&sm_info
->sit_entry_set
);
5457 init_f2fs_rwsem(&sm_info
->curseg_lock
);
5459 err
= f2fs_create_flush_cmd_control(sbi
);
5463 err
= create_discard_cmd_control(sbi
);
5467 err
= build_sit_info(sbi
);
5470 err
= build_free_segmap(sbi
);
5473 err
= build_curseg(sbi
);
5477 /* reinit free segmap based on SIT */
5478 err
= build_sit_entries(sbi
);
5482 init_free_segmap(sbi
);
5483 err
= build_dirty_segmap(sbi
);
5487 err
= sanity_check_curseg(sbi
);
5491 init_min_max_mtime(sbi
);
5495 static void discard_dirty_segmap(struct f2fs_sb_info
*sbi
,
5496 enum dirty_type dirty_type
)
5498 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
5500 mutex_lock(&dirty_i
->seglist_lock
);
5501 kvfree(dirty_i
->dirty_segmap
[dirty_type
]);
5502 dirty_i
->nr_dirty
[dirty_type
] = 0;
5503 mutex_unlock(&dirty_i
->seglist_lock
);
5506 static void destroy_victim_secmap(struct f2fs_sb_info
*sbi
)
5508 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
5510 kvfree(dirty_i
->pinned_secmap
);
5511 kvfree(dirty_i
->victim_secmap
);
5514 static void destroy_dirty_segmap(struct f2fs_sb_info
*sbi
)
5516 struct dirty_seglist_info
*dirty_i
= DIRTY_I(sbi
);
5522 /* discard pre-free/dirty segments list */
5523 for (i
= 0; i
< NR_DIRTY_TYPE
; i
++)
5524 discard_dirty_segmap(sbi
, i
);
5526 if (__is_large_section(sbi
)) {
5527 mutex_lock(&dirty_i
->seglist_lock
);
5528 kvfree(dirty_i
->dirty_secmap
);
5529 mutex_unlock(&dirty_i
->seglist_lock
);
5532 destroy_victim_secmap(sbi
);
5533 SM_I(sbi
)->dirty_info
= NULL
;
5537 static void destroy_curseg(struct f2fs_sb_info
*sbi
)
5539 struct curseg_info
*array
= SM_I(sbi
)->curseg_array
;
5544 SM_I(sbi
)->curseg_array
= NULL
;
5545 for (i
= 0; i
< NR_CURSEG_TYPE
; i
++) {
5546 kfree(array
[i
].sum_blk
);
5547 kfree(array
[i
].journal
);
5552 static void destroy_free_segmap(struct f2fs_sb_info
*sbi
)
5554 struct free_segmap_info
*free_i
= SM_I(sbi
)->free_info
;
5558 SM_I(sbi
)->free_info
= NULL
;
5559 kvfree(free_i
->free_segmap
);
5560 kvfree(free_i
->free_secmap
);
5564 static void destroy_sit_info(struct f2fs_sb_info
*sbi
)
5566 struct sit_info
*sit_i
= SIT_I(sbi
);
5571 if (sit_i
->sentries
)
5572 kvfree(sit_i
->bitmap
);
5573 kfree(sit_i
->tmp_map
);
5575 kvfree(sit_i
->sentries
);
5576 kvfree(sit_i
->sec_entries
);
5577 kvfree(sit_i
->dirty_sentries_bitmap
);
5579 SM_I(sbi
)->sit_info
= NULL
;
5580 kvfree(sit_i
->sit_bitmap
);
5581 #ifdef CONFIG_F2FS_CHECK_FS
5582 kvfree(sit_i
->sit_bitmap_mir
);
5583 kvfree(sit_i
->invalid_segmap
);
5588 void f2fs_destroy_segment_manager(struct f2fs_sb_info
*sbi
)
5590 struct f2fs_sm_info
*sm_info
= SM_I(sbi
);
5594 f2fs_destroy_flush_cmd_control(sbi
, true);
5595 destroy_discard_cmd_control(sbi
);
5596 destroy_dirty_segmap(sbi
);
5597 destroy_curseg(sbi
);
5598 destroy_free_segmap(sbi
);
5599 destroy_sit_info(sbi
);
5600 sbi
->sm_info
= NULL
;
5604 int __init
f2fs_create_segment_manager_caches(void)
5606 discard_entry_slab
= f2fs_kmem_cache_create("f2fs_discard_entry",
5607 sizeof(struct discard_entry
));
5608 if (!discard_entry_slab
)
5611 discard_cmd_slab
= f2fs_kmem_cache_create("f2fs_discard_cmd",
5612 sizeof(struct discard_cmd
));
5613 if (!discard_cmd_slab
)
5614 goto destroy_discard_entry
;
5616 sit_entry_set_slab
= f2fs_kmem_cache_create("f2fs_sit_entry_set",
5617 sizeof(struct sit_entry_set
));
5618 if (!sit_entry_set_slab
)
5619 goto destroy_discard_cmd
;
5621 revoke_entry_slab
= f2fs_kmem_cache_create("f2fs_revoke_entry",
5622 sizeof(struct revoke_entry
));
5623 if (!revoke_entry_slab
)
5624 goto destroy_sit_entry_set
;
5627 destroy_sit_entry_set
:
5628 kmem_cache_destroy(sit_entry_set_slab
);
5629 destroy_discard_cmd
:
5630 kmem_cache_destroy(discard_cmd_slab
);
5631 destroy_discard_entry
:
5632 kmem_cache_destroy(discard_entry_slab
);
5637 void f2fs_destroy_segment_manager_caches(void)
5639 kmem_cache_destroy(sit_entry_set_slab
);
5640 kmem_cache_destroy(discard_cmd_slab
);
5641 kmem_cache_destroy(discard_entry_slab
);
5642 kmem_cache_destroy(revoke_entry_slab
);