linux/fs/fs-writeback.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * fs/fs-writeback.c
   4 *
   5 * Copyright (C) 2002, Linus Torvalds.
   6 *
   7 * Contains all the functions related to writing back and waiting
   8 * upon dirty inodes against superblocks, and writing back dirty
   9 * pages against inodes.  ie: data writeback.  Writeout of the
  10 * inode itself is not handled here.
  11 *
  12 * 10Apr2002    Andrew Morton
  13 *              Split out of fs/inode.c
  14 *              Additions for address_space-based writeback
  15 */
  16
  17#include <linux/kernel.h>
  18#include <linux/export.h>
  19#include <linux/spinlock.h>
  20#include <linux/slab.h>
  21#include <linux/sched.h>
  22#include <linux/fs.h>
  23#include <linux/mm.h>
  24#include <linux/pagemap.h>
  25#include <linux/kthread.h>
  26#include <linux/writeback.h>
  27#include <linux/blkdev.h>
  28#include <linux/backing-dev.h>
  29#include <linux/tracepoint.h>
  30#include <linux/device.h>
  31#include <linux/memcontrol.h>
  32#include "internal.h"
  33
  34/*
  35 * 4MB minimal write chunk size
  36 */
  37#define MIN_WRITEBACK_PAGES     (4096UL >> (PAGE_SHIFT - 10))
  38
  39/*
  40 * Passed into wb_writeback(), essentially a subset of writeback_control
  41 */
  42struct wb_writeback_work {
  43        long nr_pages;
  44        struct super_block *sb;
  45        enum writeback_sync_modes sync_mode;
  46        unsigned int tagged_writepages:1;
  47        unsigned int for_kupdate:1;
  48        unsigned int range_cyclic:1;
  49        unsigned int for_background:1;
  50        unsigned int for_sync:1;        /* sync(2) WB_SYNC_ALL writeback */
  51        unsigned int auto_free:1;       /* free on completion */
  52        enum wb_reason reason;          /* why was writeback initiated? */
  53
  54        struct list_head list;          /* pending work list */
  55        struct wb_completion *done;     /* set if the caller waits */
  56};
  57
  58/*
  59 * If an inode is constantly having its pages dirtied, but then the
  60 * updates stop dirtytime_expire_interval seconds in the past, it's
  61 * possible for the worst case time between when an inode has its
  62 * timestamps updated and when they finally get written out to be two
  63 * dirtytime_expire_intervals.  We set the default to 12 hours (in
  64 * seconds), which means most of the time inodes will have their
  65 * timestamps written to disk after 12 hours, but in the worst case a
  66 * few inodes might not their timestamps updated for 24 hours.
  67 */
  68unsigned int dirtytime_expire_interval = 12 * 60 * 60;
  69
  70static inline struct inode *wb_inode(struct list_head *head)
  71{
  72        return list_entry(head, struct inode, i_io_list);
  73}
  74
  75/*
  76 * Include the creation of the trace points after defining the
  77 * wb_writeback_work structure and inline functions so that the definition
  78 * remains local to this file.
  79 */
  80#define CREATE_TRACE_POINTS
  81#include <trace/events/writeback.h>
  82
  83EXPORT_TRACEPOINT_SYMBOL_GPL(wbc_writepage);
  84
  85static bool wb_io_lists_populated(struct bdi_writeback *wb)
  86{
  87        if (wb_has_dirty_io(wb)) {
  88                return false;
  89        } else {
  90                set_bit(WB_has_dirty_io, &wb->state);
  91                WARN_ON_ONCE(!wb->avg_write_bandwidth);
  92                atomic_long_add(wb->avg_write_bandwidth,
  93                                &wb->bdi->tot_write_bandwidth);
  94                return true;
  95        }
  96}
  97
  98static void wb_io_lists_depopulated(struct bdi_writeback *wb)
  99{
 100        if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) &&
 101            list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) {
 102                clear_bit(WB_has_dirty_io, &wb->state);
 103                WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth,
 104                                        &wb->bdi->tot_write_bandwidth) < 0);
 105        }
 106}
 107
 108/**
 109 * inode_io_list_move_locked - move an inode onto a bdi_writeback IO list
 110 * @inode: inode to be moved
 111 * @wb: target bdi_writeback
 112 * @head: one of @wb->b_{dirty|io|more_io|dirty_time}
 113 *
 114 * Move @inode->i_io_list to @list of @wb and set %WB_has_dirty_io.
 115 * Returns %true if @inode is the first occupant of the !dirty_time IO
 116 * lists; otherwise, %false.
 117 */
 118static bool inode_io_list_move_locked(struct inode *inode,
 119                                      struct bdi_writeback *wb,
 120                                      struct list_head *head)
 121{
 122        assert_spin_locked(&wb->list_lock);
 123
 124        list_move(&inode->i_io_list, head);
 125
 126        /* dirty_time doesn't count as dirty_io until expiration */
 127        if (head != &wb->b_dirty_time)
 128                return wb_io_lists_populated(wb);
 129
 130        wb_io_lists_depopulated(wb);
 131        return false;
 132}
 133
 134/**
 135 * inode_io_list_del_locked - remove an inode from its bdi_writeback IO list
 136 * @inode: inode to be removed
 137 * @wb: bdi_writeback @inode is being removed from
 138 *
 139 * Remove @inode which may be on one of @wb->b_{dirty|io|more_io} lists and
 140 * clear %WB_has_dirty_io if all are empty afterwards.
 141 */
 142static void inode_io_list_del_locked(struct inode *inode,
 143                                     struct bdi_writeback *wb)
 144{
 145        assert_spin_locked(&wb->list_lock);
 146        assert_spin_locked(&inode->i_lock);
 147
 148        inode->i_state &= ~I_SYNC_QUEUED;
 149        list_del_init(&inode->i_io_list);
 150        wb_io_lists_depopulated(wb);
 151}
 152
 153static void wb_wakeup(struct bdi_writeback *wb)
 154{
 155        spin_lock_bh(&wb->work_lock);
 156        if (test_bit(WB_registered, &wb->state))
 157                mod_delayed_work(bdi_wq, &wb->dwork, 0);
 158        spin_unlock_bh(&wb->work_lock);
 159}
 160
 161static void finish_writeback_work(struct bdi_writeback *wb,
 162                                  struct wb_writeback_work *work)
 163{
 164        struct wb_completion *done = work->done;
 165
 166        if (work->auto_free)
 167                kfree(work);
 168        if (done) {
 169                wait_queue_head_t *waitq = done->waitq;
 170
 171                /* @done can't be accessed after the following dec */
 172                if (atomic_dec_and_test(&done->cnt))
 173                        wake_up_all(waitq);
 174        }
 175}
 176
 177static void wb_queue_work(struct bdi_writeback *wb,
 178                          struct wb_writeback_work *work)
 179{
 180        trace_writeback_queue(wb, work);
 181
 182        if (work->done)
 183                atomic_inc(&work->done->cnt);
 184
 185        spin_lock_bh(&wb->work_lock);
 186
 187        if (test_bit(WB_registered, &wb->state)) {
 188                list_add_tail(&work->list, &wb->work_list);
 189                mod_delayed_work(bdi_wq, &wb->dwork, 0);
 190        } else
 191                finish_writeback_work(wb, work);
 192
 193        spin_unlock_bh(&wb->work_lock);
 194}
 195
 196/**
 197 * wb_wait_for_completion - wait for completion of bdi_writeback_works
 198 * @done: target wb_completion
 199 *
 200 * Wait for one or more work items issued to @bdi with their ->done field
 201 * set to @done, which should have been initialized with
 202 * DEFINE_WB_COMPLETION().  This function returns after all such work items
 203 * are completed.  Work items which are waited upon aren't freed
 204 * automatically on completion.
 205 */
 206void wb_wait_for_completion(struct wb_completion *done)
 207{
 208        atomic_dec(&done->cnt);         /* put down the initial count */
 209        wait_event(*done->waitq, !atomic_read(&done->cnt));
 210}
 211
 212#ifdef CONFIG_CGROUP_WRITEBACK
 213
 214/*
 215 * Parameters for foreign inode detection, see wbc_detach_inode() to see
 216 * how they're used.
 217 *
 218 * These paramters are inherently heuristical as the detection target
 219 * itself is fuzzy.  All we want to do is detaching an inode from the
 220 * current owner if it's being written to by some other cgroups too much.
 221 *
 222 * The current cgroup writeback is built on the assumption that multiple
 223 * cgroups writing to the same inode concurrently is very rare and a mode
 224 * of operation which isn't well supported.  As such, the goal is not
 225 * taking too long when a different cgroup takes over an inode while
 226 * avoiding too aggressive flip-flops from occasional foreign writes.
 227 *
 228 * We record, very roughly, 2s worth of IO time history and if more than
 229 * half of that is foreign, trigger the switch.  The recording is quantized
 230 * to 16 slots.  To avoid tiny writes from swinging the decision too much,
 231 * writes smaller than 1/8 of avg size are ignored.
 232 */
 233#define WB_FRN_TIME_SHIFT       13      /* 1s = 2^13, upto 8 secs w/ 16bit */
 234#define WB_FRN_TIME_AVG_SHIFT   3       /* avg = avg * 7/8 + new * 1/8 */
 235#define WB_FRN_TIME_CUT_DIV     8       /* ignore rounds < avg / 8 */
 236#define WB_FRN_TIME_PERIOD      (2 * (1 << WB_FRN_TIME_SHIFT))  /* 2s */
 237
 238#define WB_FRN_HIST_SLOTS       16      /* inode->i_wb_frn_history is 16bit */
 239#define WB_FRN_HIST_UNIT        (WB_FRN_TIME_PERIOD / WB_FRN_HIST_SLOTS)
 240                                        /* each slot's duration is 2s / 16 */
 241#define WB_FRN_HIST_THR_SLOTS   (WB_FRN_HIST_SLOTS / 2)
 242                                        /* if foreign slots >= 8, switch */
 243#define WB_FRN_HIST_MAX_SLOTS   (WB_FRN_HIST_THR_SLOTS / 2 + 1)
 244                                        /* one round can affect upto 5 slots */
 245#define WB_FRN_MAX_IN_FLIGHT    1024    /* don't queue too many concurrently */
 246
 247static atomic_t isw_nr_in_flight = ATOMIC_INIT(0);
 248static struct workqueue_struct *isw_wq;
 249
 250void __inode_attach_wb(struct inode *inode, struct page *page)
 251{
 252        struct backing_dev_info *bdi = inode_to_bdi(inode);
 253        struct bdi_writeback *wb = NULL;
 254
 255        if (inode_cgwb_enabled(inode)) {
 256                struct cgroup_subsys_state *memcg_css;
 257
 258                if (page) {
 259                        memcg_css = mem_cgroup_css_from_page(page);
 260                        wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
 261                } else {
 262                        /* must pin memcg_css, see wb_get_create() */
 263                        memcg_css = task_get_css(current, memory_cgrp_id);
 264                        wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
 265                        css_put(memcg_css);
 266                }
 267        }
 268
 269        if (!wb)
 270                wb = &bdi->wb;
 271
 272        /*
 273         * There may be multiple instances of this function racing to
 274         * update the same inode.  Use cmpxchg() to tell the winner.
 275         */
 276        if (unlikely(cmpxchg(&inode->i_wb, NULL, wb)))
 277                wb_put(wb);
 278}
 279EXPORT_SYMBOL_GPL(__inode_attach_wb);
 280
 281/**
 282 * locked_inode_to_wb_and_lock_list - determine a locked inode's wb and lock it
 283 * @inode: inode of interest with i_lock held
 284 *
 285 * Returns @inode's wb with its list_lock held.  @inode->i_lock must be
 286 * held on entry and is released on return.  The returned wb is guaranteed
 287 * to stay @inode's associated wb until its list_lock is released.
 288 */
 289static struct bdi_writeback *
 290locked_inode_to_wb_and_lock_list(struct inode *inode)
 291        __releases(&inode->i_lock)
 292        __acquires(&wb->list_lock)
 293{
 294        while (true) {
 295                struct bdi_writeback *wb = inode_to_wb(inode);
 296
 297                /*
 298                 * inode_to_wb() association is protected by both
 299                 * @inode->i_lock and @wb->list_lock but list_lock nests
 300                 * outside i_lock.  Drop i_lock and verify that the
 301                 * association hasn't changed after acquiring list_lock.
 302                 */
 303                wb_get(wb);
 304                spin_unlock(&inode->i_lock);
 305                spin_lock(&wb->list_lock);
 306
 307                /* i_wb may have changed inbetween, can't use inode_to_wb() */
 308                if (likely(wb == inode->i_wb)) {
 309                        wb_put(wb);     /* @inode already has ref */
 310                        return wb;
 311                }
 312
 313                spin_unlock(&wb->list_lock);
 314                wb_put(wb);
 315                cpu_relax();
 316                spin_lock(&inode->i_lock);
 317        }
 318}
 319
 320/**
 321 * inode_to_wb_and_lock_list - determine an inode's wb and lock it
 322 * @inode: inode of interest
 323 *
 324 * Same as locked_inode_to_wb_and_lock_list() but @inode->i_lock isn't held
 325 * on entry.
 326 */
 327static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)
 328        __acquires(&wb->list_lock)
 329{
 330        spin_lock(&inode->i_lock);
 331        return locked_inode_to_wb_and_lock_list(inode);
 332}
 333
 334struct inode_switch_wbs_context {
 335        struct inode            *inode;
 336        struct bdi_writeback    *new_wb;
 337
 338        struct rcu_head         rcu_head;
 339        struct work_struct      work;
 340};
 341
 342static void bdi_down_write_wb_switch_rwsem(struct backing_dev_info *bdi)
 343{
 344        down_write(&bdi->wb_switch_rwsem);
 345}
 346
 347static void bdi_up_write_wb_switch_rwsem(struct backing_dev_info *bdi)
 348{
 349        up_write(&bdi->wb_switch_rwsem);
 350}
 351
 352static void inode_switch_wbs_work_fn(struct work_struct *work)
 353{
 354        struct inode_switch_wbs_context *isw =
 355                container_of(work, struct inode_switch_wbs_context, work);
 356        struct inode *inode = isw->inode;
 357        struct backing_dev_info *bdi = inode_to_bdi(inode);
 358        struct address_space *mapping = inode->i_mapping;
 359        struct bdi_writeback *old_wb = inode->i_wb;
 360        struct bdi_writeback *new_wb = isw->new_wb;
 361        XA_STATE(xas, &mapping->i_pages, 0);
 362        struct page *page;
 363        bool switched = false;
 364
 365        /*
 366         * If @inode switches cgwb membership while sync_inodes_sb() is
 367         * being issued, sync_inodes_sb() might miss it.  Synchronize.
 368         */
 369        down_read(&bdi->wb_switch_rwsem);
 370
 371        /*
 372         * By the time control reaches here, RCU grace period has passed
 373         * since I_WB_SWITCH assertion and all wb stat update transactions
 374         * between unlocked_inode_to_wb_begin/end() are guaranteed to be
 375         * synchronizing against the i_pages lock.
 376         *
 377         * Grabbing old_wb->list_lock, inode->i_lock and the i_pages lock
 378         * gives us exclusion against all wb related operations on @inode
 379         * including IO list manipulations and stat updates.
 380         */
 381        if (old_wb < new_wb) {
 382                spin_lock(&old_wb->list_lock);
 383                spin_lock_nested(&new_wb->list_lock, SINGLE_DEPTH_NESTING);
 384        } else {
 385                spin_lock(&new_wb->list_lock);
 386                spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING);
 387        }
 388        spin_lock(&inode->i_lock);
 389        xa_lock_irq(&mapping->i_pages);
 390
 391        /*
 392         * Once I_FREEING is visible under i_lock, the eviction path owns
 393         * the inode and we shouldn't modify ->i_io_list.
 394         */
 395        if (unlikely(inode->i_state & I_FREEING))
 396                goto skip_switch;
 397
 398        trace_inode_switch_wbs(inode, old_wb, new_wb);
 399
 400        /*
 401         * Count and transfer stats.  Note that PAGECACHE_TAG_DIRTY points
 402         * to possibly dirty pages while PAGECACHE_TAG_WRITEBACK points to
 403         * pages actually under writeback.
 404         */
 405        xas_for_each_marked(&xas, page, ULONG_MAX, PAGECACHE_TAG_DIRTY) {
 406                if (PageDirty(page)) {
 407                        dec_wb_stat(old_wb, WB_RECLAIMABLE);
 408                        inc_wb_stat(new_wb, WB_RECLAIMABLE);
 409                }
 410        }
 411
 412        xas_set(&xas, 0);
 413        xas_for_each_marked(&xas, page, ULONG_MAX, PAGECACHE_TAG_WRITEBACK) {
 414                WARN_ON_ONCE(!PageWriteback(page));
 415                dec_wb_stat(old_wb, WB_WRITEBACK);
 416                inc_wb_stat(new_wb, WB_WRITEBACK);
 417        }
 418
 419        wb_get(new_wb);
 420
 421        /*
 422         * Transfer to @new_wb's IO list if necessary.  The specific list
 423         * @inode was on is ignored and the inode is put on ->b_dirty which
 424         * is always correct including from ->b_dirty_time.  The transfer
 425         * preserves @inode->dirtied_when ordering.
 426         */
 427        if (!list_empty(&inode->i_io_list)) {
 428                struct inode *pos;
 429
 430                inode_io_list_del_locked(inode, old_wb);
 431                inode->i_wb = new_wb;
 432                list_for_each_entry(pos, &new_wb->b_dirty, i_io_list)
 433                        if (time_after_eq(inode->dirtied_when,
 434                                          pos->dirtied_when))
 435                                break;
 436                inode_io_list_move_locked(inode, new_wb, pos->i_io_list.prev);
 437        } else {
 438                inode->i_wb = new_wb;
 439        }
 440
 441        /* ->i_wb_frn updates may race wbc_detach_inode() but doesn't matter */
 442        inode->i_wb_frn_winner = 0;
 443        inode->i_wb_frn_avg_time = 0;
 444        inode->i_wb_frn_history = 0;
 445        switched = true;
 446skip_switch:
 447        /*
 448         * Paired with load_acquire in unlocked_inode_to_wb_begin() and
 449         * ensures that the new wb is visible if they see !I_WB_SWITCH.
 450         */
 451        smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH);
 452
 453        xa_unlock_irq(&mapping->i_pages);
 454        spin_unlock(&inode->i_lock);
 455        spin_unlock(&new_wb->list_lock);
 456        spin_unlock(&old_wb->list_lock);
 457
 458        up_read(&bdi->wb_switch_rwsem);
 459
 460        if (switched) {
 461                wb_wakeup(new_wb);
 462                wb_put(old_wb);
 463        }
 464        wb_put(new_wb);
 465
 466        iput(inode);
 467        kfree(isw);
 468
 469        atomic_dec(&isw_nr_in_flight);
 470}
 471
 472static void inode_switch_wbs_rcu_fn(struct rcu_head *rcu_head)
 473{
 474        struct inode_switch_wbs_context *isw = container_of(rcu_head,
 475                                struct inode_switch_wbs_context, rcu_head);
 476
 477        /* needs to grab bh-unsafe locks, bounce to work item */
 478        INIT_WORK(&isw->work, inode_switch_wbs_work_fn);
 479        queue_work(isw_wq, &isw->work);
 480}
 481
 482/**
 483 * inode_switch_wbs - change the wb association of an inode
 484 * @inode: target inode
 485 * @new_wb_id: ID of the new wb
 486 *
 487 * Switch @inode's wb association to the wb identified by @new_wb_id.  The
 488 * switching is performed asynchronously and may fail silently.
 489 */
 490static void inode_switch_wbs(struct inode *inode, int new_wb_id)
 491{
 492        struct backing_dev_info *bdi = inode_to_bdi(inode);
 493        struct cgroup_subsys_state *memcg_css;
 494        struct inode_switch_wbs_context *isw;
 495
 496        /* noop if seems to be already in progress */
 497        if (inode->i_state & I_WB_SWITCH)
 498                return;
 499
 500        /* avoid queueing a new switch if too many are already in flight */
 501        if (atomic_read(&isw_nr_in_flight) > WB_FRN_MAX_IN_FLIGHT)
 502                return;
 503
 504        isw = kzalloc(sizeof(*isw), GFP_ATOMIC);
 505        if (!isw)
 506                return;
 507
 508        /* find and pin the new wb */
 509        rcu_read_lock();
 510        memcg_css = css_from_id(new_wb_id, &memory_cgrp_subsys);
 511        if (memcg_css)
 512                isw->new_wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);
 513        rcu_read_unlock();
 514        if (!isw->new_wb)
 515                goto out_free;
 516
 517        /* while holding I_WB_SWITCH, no one else can update the association */
 518        spin_lock(&inode->i_lock);
 519        if (!(inode->i_sb->s_flags & SB_ACTIVE) ||
 520            inode->i_state & (I_WB_SWITCH | I_FREEING) ||
 521            inode_to_wb(inode) == isw->new_wb) {
 522                spin_unlock(&inode->i_lock);
 523                goto out_free;
 524        }
 525        inode->i_state |= I_WB_SWITCH;
 526        __iget(inode);
 527        spin_unlock(&inode->i_lock);
 528
 529        isw->inode = inode;
 530
 531        /*
 532         * In addition to synchronizing among switchers, I_WB_SWITCH tells
 533         * the RCU protected stat update paths to grab the i_page
 534         * lock so that stat transfer can synchronize against them.
 535         * Let's continue after I_WB_SWITCH is guaranteed to be visible.
 536         */
 537        call_rcu(&isw->rcu_head, inode_switch_wbs_rcu_fn);
 538
 539        atomic_inc(&isw_nr_in_flight);
 540        return;
 541
 542out_free:
 543        if (isw->new_wb)
 544                wb_put(isw->new_wb);
 545        kfree(isw);
 546}
 547
 548/**
 549 * wbc_attach_and_unlock_inode - associate wbc with target inode and unlock it
 550 * @wbc: writeback_control of interest
 551 * @inode: target inode
 552 *
 553 * @inode is locked and about to be written back under the control of @wbc.
 554 * Record @inode's writeback context into @wbc and unlock the i_lock.  On
 555 * writeback completion, wbc_detach_inode() should be called.  This is used
 556 * to track the cgroup writeback context.
 557 */
 558void wbc_attach_and_unlock_inode(struct writeback_control *wbc,
 559                                 struct inode *inode)
 560{
 561        if (!inode_cgwb_enabled(inode)) {
 562                spin_unlock(&inode->i_lock);
 563                return;
 564        }
 565
 566        wbc->wb = inode_to_wb(inode);
 567        wbc->inode = inode;
 568
 569        wbc->wb_id = wbc->wb->memcg_css->id;
 570        wbc->wb_lcand_id = inode->i_wb_frn_winner;
 571        wbc->wb_tcand_id = 0;
 572        wbc->wb_bytes = 0;
 573        wbc->wb_lcand_bytes = 0;
 574        wbc->wb_tcand_bytes = 0;
 575
 576        wb_get(wbc->wb);
 577        spin_unlock(&inode->i_lock);
 578
 579        /*
 580         * A dying wb indicates that either the blkcg associated with the
 581         * memcg changed or the associated memcg is dying.  In the first
 582         * case, a replacement wb should already be available and we should
 583         * refresh the wb immediately.  In the second case, trying to
 584         * refresh will keep failing.
 585         */
 586        if (unlikely(wb_dying(wbc->wb) && !css_is_dying(wbc->wb->memcg_css)))
 587                inode_switch_wbs(inode, wbc->wb_id);
 588}
 589EXPORT_SYMBOL_GPL(wbc_attach_and_unlock_inode);
 590
 591/**
 592 * wbc_detach_inode - disassociate wbc from inode and perform foreign detection
 593 * @wbc: writeback_control of the just finished writeback
 594 *
 595 * To be called after a writeback attempt of an inode finishes and undoes
 596 * wbc_attach_and_unlock_inode().  Can be called under any context.
 597 *
 598 * As concurrent write sharing of an inode is expected to be very rare and
 599 * memcg only tracks page ownership on first-use basis severely confining
 600 * the usefulness of such sharing, cgroup writeback tracks ownership
 601 * per-inode.  While the support for concurrent write sharing of an inode
 602 * is deemed unnecessary, an inode being written to by different cgroups at
 603 * different points in time is a lot more common, and, more importantly,
 604 * charging only by first-use can too readily lead to grossly incorrect
 605 * behaviors (single foreign page can lead to gigabytes of writeback to be
 606 * incorrectly attributed).
 607 *
 608 * To resolve this issue, cgroup writeback detects the majority dirtier of
 609 * an inode and transfers the ownership to it.  To avoid unnnecessary
 610 * oscillation, the detection mechanism keeps track of history and gives
 611 * out the switch verdict only if the foreign usage pattern is stable over
 612 * a certain amount of time and/or writeback attempts.
 613 *
 614 * On each writeback attempt, @wbc tries to detect the majority writer
 615 * using Boyer-Moore majority vote algorithm.  In addition to the byte
 616 * count from the majority voting, it also counts the bytes written for the
 617 * current wb and the last round's winner wb (max of last round's current
 618 * wb, the winner from two rounds ago, and the last round's majority
 619 * candidate).  Keeping track of the historical winner helps the algorithm
 620 * to semi-reliably detect the most active writer even when it's not the
 621 * absolute majority.
 622 *
 623 * Once the winner of the round is determined, whether the winner is
 624 * foreign or not and how much IO time the round consumed is recorded in
 625 * inode->i_wb_frn_history.  If the amount of recorded foreign IO time is
 626 * over a certain threshold, the switch verdict is given.
 627 */
 628void wbc_detach_inode(struct writeback_control *wbc)
 629{
 630        struct bdi_writeback *wb = wbc->wb;
 631        struct inode *inode = wbc->inode;
 632        unsigned long avg_time, max_bytes, max_time;
 633        u16 history;
 634        int max_id;
 635
 636        if (!wb)
 637                return;
 638
 639        history = inode->i_wb_frn_history;
 640        avg_time = inode->i_wb_frn_avg_time;
 641
 642        /* pick the winner of this round */
 643        if (wbc->wb_bytes >= wbc->wb_lcand_bytes &&
 644            wbc->wb_bytes >= wbc->wb_tcand_bytes) {
 645                max_id = wbc->wb_id;
 646                max_bytes = wbc->wb_bytes;
 647        } else if (wbc->wb_lcand_bytes >= wbc->wb_tcand_bytes) {
 648                max_id = wbc->wb_lcand_id;
 649                max_bytes = wbc->wb_lcand_bytes;
 650        } else {
 651                max_id = wbc->wb_tcand_id;
 652                max_bytes = wbc->wb_tcand_bytes;
 653        }
 654
 655        /*
 656         * Calculate the amount of IO time the winner consumed and fold it
 657         * into the running average kept per inode.  If the consumed IO
 658         * time is lower than avag / WB_FRN_TIME_CUT_DIV, ignore it for
 659         * deciding whether to switch or not.  This is to prevent one-off
 660         * small dirtiers from skewing the verdict.
 661         */
 662        max_time = DIV_ROUND_UP((max_bytes >> PAGE_SHIFT) << WB_FRN_TIME_SHIFT,
 663                                wb->avg_write_bandwidth);
 664        if (avg_time)
 665                avg_time += (max_time >> WB_FRN_TIME_AVG_SHIFT) -
 666                            (avg_time >> WB_FRN_TIME_AVG_SHIFT);
 667        else
 668                avg_time = max_time;    /* immediate catch up on first run */
 669
 670        if (max_time >= avg_time / WB_FRN_TIME_CUT_DIV) {
 671                int slots;
 672
 673                /*
 674                 * The switch verdict is reached if foreign wb's consume
 675                 * more than a certain proportion of IO time in a
 676                 * WB_FRN_TIME_PERIOD.  This is loosely tracked by 16 slot
 677                 * history mask where each bit represents one sixteenth of
 678                 * the period.  Determine the number of slots to shift into
 679                 * history from @max_time.
 680                 */
 681                slots = min(DIV_ROUND_UP(max_time, WB_FRN_HIST_UNIT),
 682                            (unsigned long)WB_FRN_HIST_MAX_SLOTS);
 683                history <<= slots;
 684                if (wbc->wb_id != max_id)
 685                        history |= (1U << slots) - 1;
 686
 687                if (history)
 688                        trace_inode_foreign_history(inode, wbc, history);
 689
 690                /*
 691                 * Switch if the current wb isn't the consistent winner.
 692                 * If there are multiple closely competing dirtiers, the
 693                 * inode may switch across them repeatedly over time, which
 694                 * is okay.  The main goal is avoiding keeping an inode on
 695                 * the wrong wb for an extended period of time.
 696                 */
 697                if (hweight32(history) > WB_FRN_HIST_THR_SLOTS)
 698                        inode_switch_wbs(inode, max_id);
 699        }
 700
 701        /*
 702         * Multiple instances of this function may race to update the
 703         * following fields but we don't mind occassional inaccuracies.
 704         */
 705        inode->i_wb_frn_winner = max_id;
 706        inode->i_wb_frn_avg_time = min(avg_time, (unsigned long)U16_MAX);
 707        inode->i_wb_frn_history = history;
 708
 709        wb_put(wbc->wb);
 710        wbc->wb = NULL;
 711}
 712EXPORT_SYMBOL_GPL(wbc_detach_inode);
 713
 714/**
 715 * wbc_account_cgroup_owner - account writeback to update inode cgroup ownership
 716 * @wbc: writeback_control of the writeback in progress
 717 * @page: page being written out
 718 * @bytes: number of bytes being written out
 719 *
 720 * @bytes from @page are about to written out during the writeback
 721 * controlled by @wbc.  Keep the book for foreign inode detection.  See
 722 * wbc_detach_inode().
 723 */
 724void wbc_account_cgroup_owner(struct writeback_control *wbc, struct page *page,
 725                              size_t bytes)
 726{
 727        struct cgroup_subsys_state *css;
 728        int id;
 729
 730        /*
 731         * pageout() path doesn't attach @wbc to the inode being written
 732         * out.  This is intentional as we don't want the function to block
 733         * behind a slow cgroup.  Ultimately, we want pageout() to kick off
 734         * regular writeback instead of writing things out itself.
 735         */
 736        if (!wbc->wb || wbc->no_cgroup_owner)
 737                return;
 738
 739        css = mem_cgroup_css_from_page(page);
 740        /* dead cgroups shouldn't contribute to inode ownership arbitration */
 741        if (!(css->flags & CSS_ONLINE))
 742                return;
 743
 744        id = css->id;
 745
 746        if (id == wbc->wb_id) {
 747                wbc->wb_bytes += bytes;
 748                return;
 749        }
 750
 751        if (id == wbc->wb_lcand_id)
 752                wbc->wb_lcand_bytes += bytes;
 753
 754        /* Boyer-Moore majority vote algorithm */
 755        if (!wbc->wb_tcand_bytes)
 756                wbc->wb_tcand_id = id;
 757        if (id == wbc->wb_tcand_id)
 758                wbc->wb_tcand_bytes += bytes;
 759        else
 760                wbc->wb_tcand_bytes -= min(bytes, wbc->wb_tcand_bytes);
 761}
 762EXPORT_SYMBOL_GPL(wbc_account_cgroup_owner);
 763
 764/**
 765 * inode_congested - test whether an inode is congested
 766 * @inode: inode to test for congestion (may be NULL)
 767 * @cong_bits: mask of WB_[a]sync_congested bits to test
 768 *
 769 * Tests whether @inode is congested.  @cong_bits is the mask of congestion
 770 * bits to test and the return value is the mask of set bits.
 771 *
 772 * If cgroup writeback is enabled for @inode, the congestion state is
 773 * determined by whether the cgwb (cgroup bdi_writeback) for the blkcg
 774 * associated with @inode is congested; otherwise, the root wb's congestion
 775 * state is used.
 776 *
 777 * @inode is allowed to be NULL as this function is often called on
 778 * mapping->host which is NULL for the swapper space.
 779 */
 780int inode_congested(struct inode *inode, int cong_bits)
 781{
 782        /*
 783         * Once set, ->i_wb never becomes NULL while the inode is alive.
 784         * Start transaction iff ->i_wb is visible.
 785         */
 786        if (inode && inode_to_wb_is_valid(inode)) {
 787                struct bdi_writeback *wb;
 788                struct wb_lock_cookie lock_cookie = {};
 789                bool congested;
 790
 791                wb = unlocked_inode_to_wb_begin(inode, &lock_cookie);
 792                congested = wb_congested(wb, cong_bits);
 793                unlocked_inode_to_wb_end(inode, &lock_cookie);
 794                return congested;
 795        }
 796
 797        return wb_congested(&inode_to_bdi(inode)->wb, cong_bits);
 798}
 799EXPORT_SYMBOL_GPL(inode_congested);
 800
 801/**
 802 * wb_split_bdi_pages - split nr_pages to write according to bandwidth
 803 * @wb: target bdi_writeback to split @nr_pages to
 804 * @nr_pages: number of pages to write for the whole bdi
 805 *
 806 * Split @wb's portion of @nr_pages according to @wb's write bandwidth in
 807 * relation to the total write bandwidth of all wb's w/ dirty inodes on
 808 * @wb->bdi.
 809 */
 810static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)
 811{
 812        unsigned long this_bw = wb->avg_write_bandwidth;
 813        unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);
 814
 815        if (nr_pages == LONG_MAX)
 816                return LONG_MAX;
 817
 818        /*
 819         * This may be called on clean wb's and proportional distribution
 820         * may not make sense, just use the original @nr_pages in those
 821         * cases.  In general, we wanna err on the side of writing more.
 822         */
 823        if (!tot_bw || this_bw >= tot_bw)
 824                return nr_pages;
 825        else
 826                return DIV_ROUND_UP_ULL((u64)nr_pages * this_bw, tot_bw);
 827}
 828
 829/**
 830 * bdi_split_work_to_wbs - split a wb_writeback_work to all wb's of a bdi
 831 * @bdi: target backing_dev_info
 832 * @base_work: wb_writeback_work to issue
 833 * @skip_if_busy: skip wb's which already have writeback in progress
 834 *
 835 * Split and issue @base_work to all wb's (bdi_writeback's) of @bdi which
 836 * have dirty inodes.  If @base_work->nr_page isn't %LONG_MAX, it's
 837 * distributed to the busy wbs according to each wb's proportion in the
 838 * total active write bandwidth of @bdi.
 839 */
 840static void bdi_split_work_to_wbs(struct backing_dev_info *bdi,
 841                                  struct wb_writeback_work *base_work,
 842                                  bool skip_if_busy)
 843{
 844        struct bdi_writeback *last_wb = NULL;
 845        struct bdi_writeback *wb = list_entry(&bdi->wb_list,
 846                                              struct bdi_writeback, bdi_node);
 847
 848        might_sleep();
 849restart:
 850        rcu_read_lock();
 851        list_for_each_entry_continue_rcu(wb, &bdi->wb_list, bdi_node) {
 852                DEFINE_WB_COMPLETION(fallback_work_done, bdi);
 853                struct wb_writeback_work fallback_work;
 854                struct wb_writeback_work *work;
 855                long nr_pages;
 856
 857                if (last_wb) {
 858                        wb_put(last_wb);
 859                        last_wb = NULL;
 860                }
 861
 862                /* SYNC_ALL writes out I_DIRTY_TIME too */
 863                if (!wb_has_dirty_io(wb) &&
 864                    (base_work->sync_mode == WB_SYNC_NONE ||
 865                     list_empty(&wb->b_dirty_time)))
 866                        continue;
 867                if (skip_if_busy && writeback_in_progress(wb))
 868                        continue;
 869
 870                nr_pages = wb_split_bdi_pages(wb, base_work->nr_pages);
 871
 872                work = kmalloc(sizeof(*work), GFP_ATOMIC);
 873                if (work) {
 874                        *work = *base_work;
 875                        work->nr_pages = nr_pages;
 876                        work->auto_free = 1;
 877                        wb_queue_work(wb, work);
 878                        continue;
 879                }
 880
 881                /* alloc failed, execute synchronously using on-stack fallback */
 882                work = &fallback_work;
 883                *work = *base_work;
 884                work->nr_pages = nr_pages;
 885                work->auto_free = 0;
 886                work->done = &fallback_work_done;
 887
 888                wb_queue_work(wb, work);
 889
 890                /*
 891                 * Pin @wb so that it stays on @bdi->wb_list.  This allows
 892                 * continuing iteration from @wb after dropping and
 893                 * regrabbing rcu read lock.
 894                 */
 895                wb_get(wb);
 896                last_wb = wb;
 897
 898                rcu_read_unlock();
 899                wb_wait_for_completion(&fallback_work_done);
 900                goto restart;
 901        }
 902        rcu_read_unlock();
 903
 904        if (last_wb)
 905                wb_put(last_wb);
 906}
 907
 908/**
 909 * cgroup_writeback_by_id - initiate cgroup writeback from bdi and memcg IDs
 910 * @bdi_id: target bdi id
 911 * @memcg_id: target memcg css id
 912 * @nr: number of pages to write, 0 for best-effort dirty flushing
 913 * @reason: reason why some writeback work initiated
 914 * @done: target wb_completion
 915 *
 916 * Initiate flush of the bdi_writeback identified by @bdi_id and @memcg_id
 917 * with the specified parameters.
 918 */
 919int cgroup_writeback_by_id(u64 bdi_id, int memcg_id, unsigned long nr,
 920                           enum wb_reason reason, struct wb_completion *done)
 921{
 922        struct backing_dev_info *bdi;
 923        struct cgroup_subsys_state *memcg_css;
 924        struct bdi_writeback *wb;
 925        struct wb_writeback_work *work;
 926        int ret;
 927
 928        /* lookup bdi and memcg */
 929        bdi = bdi_get_by_id(bdi_id);
 930        if (!bdi)
 931                return -ENOENT;
 932
 933        rcu_read_lock();
 934        memcg_css = css_from_id(memcg_id, &memory_cgrp_subsys);
 935        if (memcg_css && !css_tryget(memcg_css))
 936                memcg_css = NULL;
 937        rcu_read_unlock();
 938        if (!memcg_css) {
 939                ret = -ENOENT;
 940                goto out_bdi_put;
 941        }
 942
 943        /*
 944         * And find the associated wb.  If the wb isn't there already
 945         * there's nothing to flush, don't create one.
 946         */
 947        wb = wb_get_lookup(bdi, memcg_css);
 948        if (!wb) {
 949                ret = -ENOENT;
 950                goto out_css_put;
 951        }
 952
 953        /*
 954         * If @nr is zero, the caller is attempting to write out most of
 955         * the currently dirty pages.  Let's take the current dirty page
 956         * count and inflate it by 25% which should be large enough to
 957         * flush out most dirty pages while avoiding getting livelocked by
 958         * concurrent dirtiers.
 959         */
 960        if (!nr) {
 961                unsigned long filepages, headroom, dirty, writeback;
 962
 963                mem_cgroup_wb_stats(wb, &filepages, &headroom, &dirty,
 964                                      &writeback);
 965                nr = dirty * 10 / 8;
 966        }
 967
 968        /* issue the writeback work */
 969        work = kzalloc(sizeof(*work), GFP_NOWAIT | __GFP_NOWARN);
 970        if (work) {
 971                work->nr_pages = nr;
 972                work->sync_mode = WB_SYNC_NONE;
 973                work->range_cyclic = 1;
 974                work->reason = reason;
 975                work->done = done;
 976                work->auto_free = 1;
 977                wb_queue_work(wb, work);
 978                ret = 0;
 979        } else {
 980                ret = -ENOMEM;
 981        }
 982
 983        wb_put(wb);
 984out_css_put:
 985        css_put(memcg_css);
 986out_bdi_put:
 987        bdi_put(bdi);
 988        return ret;
 989}
 990
 991/**
 992 * cgroup_writeback_umount - flush inode wb switches for umount
 993 *
 994 * This function is called when a super_block is about to be destroyed and
 995 * flushes in-flight inode wb switches.  An inode wb switch goes through
 996 * RCU and then workqueue, so the two need to be flushed in order to ensure
 997 * that all previously scheduled switches are finished.  As wb switches are
 998 * rare occurrences and synchronize_rcu() can take a while, perform
 999 * flushing iff wb switches are in flight.
1000 */
1001void cgroup_writeback_umount(void)
1002{
1003        if (atomic_read(&isw_nr_in_flight)) {
1004                /*
1005                 * Use rcu_barrier() to wait for all pending callbacks to
1006                 * ensure that all in-flight wb switches are in the workqueue.
1007                 */
1008                rcu_barrier();
1009                flush_workqueue(isw_wq);
1010        }
1011}
1012
1013static int __init cgroup_writeback_init(void)
1014{
1015        isw_wq = alloc_workqueue("inode_switch_wbs", 0, 0);
1016        if (!isw_wq)
1017                return -ENOMEM;
1018        return 0;
1019}
1020fs_initcall(cgroup_writeback_init);
1021
1022#else   /* CONFIG_CGROUP_WRITEBACK */
1023
1024static void bdi_down_write_wb_switch_rwsem(struct backing_dev_info *bdi) { }
1025static void bdi_up_write_wb_switch_rwsem(struct backing_dev_info *bdi) { }
1026
1027static struct bdi_writeback *
1028locked_inode_to_wb_and_lock_list(struct inode *inode)
1029        __releases(&inode->i_lock)
1030        __acquires(&wb->list_lock)
1031{
1032        struct bdi_writeback *wb = inode_to_wb(inode);
1033
1034        spin_unlock(&inode->i_lock);
1035        spin_lock(&wb->list_lock);
1036        return wb;
1037}
1038
1039static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)
1040        __acquires(&wb->list_lock)
1041{
1042        struct bdi_writeback *wb = inode_to_wb(inode);
1043
1044        spin_lock(&wb->list_lock);
1045        return wb;
1046}
1047
1048static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)
1049{
1050        return nr_pages;
1051}
1052
1053static void bdi_split_work_to_wbs(struct backing_dev_info *bdi,
1054                                  struct wb_writeback_work *base_work,
1055                                  bool skip_if_busy)
1056{
1057        might_sleep();
1058
1059        if (!skip_if_busy || !writeback_in_progress(&bdi->wb)) {
1060                base_work->auto_free = 0;
1061                wb_queue_work(&bdi->wb, base_work);
1062        }
1063}
1064
1065#endif  /* CONFIG_CGROUP_WRITEBACK */
1066
1067/*
1068 * Add in the number of potentially dirty inodes, because each inode
1069 * write can dirty pagecache in the underlying blockdev.
1070 */
1071static unsigned long get_nr_dirty_pages(void)
1072{
1073        return global_node_page_state(NR_FILE_DIRTY) +
1074                get_nr_dirty_inodes();
1075}
1076
1077static void wb_start_writeback(struct bdi_writeback *wb, enum wb_reason reason)
1078{
1079        if (!wb_has_dirty_io(wb))
1080                return;
1081
1082        /*
1083         * All callers of this function want to start writeback of all
1084         * dirty pages. Places like vmscan can call this at a very
1085         * high frequency, causing pointless allocations of tons of
1086         * work items and keeping the flusher threads busy retrieving
1087         * that work. Ensure that we only allow one of them pending and
1088         * inflight at the time.
1089         */
1090        if (test_bit(WB_start_all, &wb->state) ||
1091            test_and_set_bit(WB_start_all, &wb->state))
1092                return;
1093
1094        wb->start_all_reason = reason;
1095        wb_wakeup(wb);
1096}
1097
1098/**
1099 * wb_start_background_writeback - start background writeback
1100 * @wb: bdi_writback to write from
1101 *
1102 * Description:
1103 *   This makes sure WB_SYNC_NONE background writeback happens. When
1104 *   this function returns, it is only guaranteed that for given wb
1105 *   some IO is happening if we are over background dirty threshold.
1106 *   Caller need not hold sb s_umount semaphore.
1107 */
1108void wb_start_background_writeback(struct bdi_writeback *wb)
1109{
1110        /*
1111         * We just wake up the flusher thread. It will perform background
1112         * writeback as soon as there is no other work to do.
1113         */
1114        trace_writeback_wake_background(wb);
1115        wb_wakeup(wb);
1116}
1117
1118/*
1119 * Remove the inode from the writeback list it is on.
1120 */
1121void inode_io_list_del(struct inode *inode)
1122{
1123        struct bdi_writeback *wb;
1124
1125        wb = inode_to_wb_and_lock_list(inode);
1126        spin_lock(&inode->i_lock);
1127        inode_io_list_del_locked(inode, wb);
1128        spin_unlock(&inode->i_lock);
1129        spin_unlock(&wb->list_lock);
1130}
1131EXPORT_SYMBOL(inode_io_list_del);
1132
1133/*
1134 * mark an inode as under writeback on the sb
1135 */
1136void sb_mark_inode_writeback(struct inode *inode)
1137{
1138        struct super_block *sb = inode->i_sb;
1139        unsigned long flags;
1140
1141        if (list_empty(&inode->i_wb_list)) {
1142                spin_lock_irqsave(&sb->s_inode_wblist_lock, flags);
1143                if (list_empty(&inode->i_wb_list)) {
1144                        list_add_tail(&inode->i_wb_list, &sb->s_inodes_wb);
1145                        trace_sb_mark_inode_writeback(inode);
1146                }
1147                spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags);
1148        }
1149}
1150
1151/*
1152 * clear an inode as under writeback on the sb
1153 */
1154void sb_clear_inode_writeback(struct inode *inode)
1155{
1156        struct super_block *sb = inode->i_sb;
1157        unsigned long flags;
1158
1159        if (!list_empty(&inode->i_wb_list)) {
1160                spin_lock_irqsave(&sb->s_inode_wblist_lock, flags);
1161                if (!list_empty(&inode->i_wb_list)) {
1162                        list_del_init(&inode->i_wb_list);
1163                        trace_sb_clear_inode_writeback(inode);
1164                }
1165                spin_unlock_irqrestore(&sb->s_inode_wblist_lock, flags);
1166        }
1167}
1168
1169/*
1170 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
1171 * furthest end of its superblock's dirty-inode list.
1172 *
1173 * Before stamping the inode's ->dirtied_when, we check to see whether it is
1174 * already the most-recently-dirtied inode on the b_dirty list.  If that is
1175 * the case then the inode must have been redirtied while it was being written
1176 * out and we don't reset its dirtied_when.
1177 */
1178static void redirty_tail_locked(struct inode *inode, struct bdi_writeback *wb)
1179{
1180        assert_spin_locked(&inode->i_lock);
1181
1182        if (!list_empty(&wb->b_dirty)) {
1183                struct inode *tail;
1184
1185                tail = wb_inode(wb->b_dirty.next);
1186                if (time_before(inode->dirtied_when, tail->dirtied_when))
1187                        inode->dirtied_when = jiffies;
1188        }
1189        inode_io_list_move_locked(inode, wb, &wb->b_dirty);
1190        inode->i_state &= ~I_SYNC_QUEUED;
1191}
1192
1193static void redirty_tail(struct inode *inode, struct bdi_writeback *wb)
1194{
1195        spin_lock(&inode->i_lock);
1196        redirty_tail_locked(inode, wb);
1197        spin_unlock(&inode->i_lock);
1198}
1199
1200/*
1201 * requeue inode for re-scanning after bdi->b_io list is exhausted.
1202 */
1203static void requeue_io(struct inode *inode, struct bdi_writeback *wb)
1204{
1205        inode_io_list_move_locked(inode, wb, &wb->b_more_io);
1206}
1207
1208static void inode_sync_complete(struct inode *inode)
1209{
1210        inode->i_state &= ~I_SYNC;
1211        /* If inode is clean an unused, put it into LRU now... */
1212        inode_add_lru(inode);
1213        /* Waiters must see I_SYNC cleared before being woken up */
1214        smp_mb();
1215        wake_up_bit(&inode->i_state, __I_SYNC);
1216}
1217
1218static bool inode_dirtied_after(struct inode *inode, unsigned long t)
1219{
1220        bool ret = time_after(inode->dirtied_when, t);
1221#ifndef CONFIG_64BIT
1222        /*
1223         * For inodes being constantly redirtied, dirtied_when can get stuck.
1224         * It _appears_ to be in the future, but is actually in distant past.
1225         * This test is necessary to prevent such wrapped-around relative times
1226         * from permanently stopping the whole bdi writeback.
1227         */
1228        ret = ret && time_before_eq(inode->dirtied_when, jiffies);
1229#endif
1230        return ret;
1231}
1232
1233#define EXPIRE_DIRTY_ATIME 0x0001
1234
1235/*
1236 * Move expired (dirtied before dirtied_before) dirty inodes from
1237 * @delaying_queue to @dispatch_queue.
1238 */
1239static int move_expired_inodes(struct list_head *delaying_queue,
1240                               struct list_head *dispatch_queue,
1241                               unsigned long dirtied_before)
1242{
1243        LIST_HEAD(tmp);
1244        struct list_head *pos, *node;
1245        struct super_block *sb = NULL;
1246        struct inode *inode;
1247        int do_sb_sort = 0;
1248        int moved = 0;
1249
1250        while (!list_empty(delaying_queue)) {
1251                inode = wb_inode(delaying_queue->prev);
1252                if (inode_dirtied_after(inode, dirtied_before))
1253                        break;
1254                list_move(&inode->i_io_list, &tmp);
1255                moved++;
1256                spin_lock(&inode->i_lock);
1257                inode->i_state |= I_SYNC_QUEUED;
1258                spin_unlock(&inode->i_lock);
1259                if (sb_is_blkdev_sb(inode->i_sb))
1260                        continue;
1261                if (sb && sb != inode->i_sb)
1262                        do_sb_sort = 1;
1263                sb = inode->i_sb;
1264        }
1265
1266        /* just one sb in list, splice to dispatch_queue and we're done */
1267        if (!do_sb_sort) {
1268                list_splice(&tmp, dispatch_queue);
1269                goto out;
1270        }
1271
1272        /* Move inodes from one superblock together */
1273        while (!list_empty(&tmp)) {
1274                sb = wb_inode(tmp.prev)->i_sb;
1275                list_for_each_prev_safe(pos, node, &tmp) {
1276                        inode = wb_inode(pos);
1277                        if (inode->i_sb == sb)
1278                                list_move(&inode->i_io_list, dispatch_queue);
1279                }
1280        }
1281out:
1282        return moved;
1283}
1284
1285/*
1286 * Queue all expired dirty inodes for io, eldest first.
1287 * Before
1288 *         newly dirtied     b_dirty    b_io    b_more_io
1289 *         =============>    gf         edc     BA
1290 * After
1291 *         newly dirtied     b_dirty    b_io    b_more_io
1292 *         =============>    g          fBAedc
1293 *                                           |
1294 *                                           +--> dequeue for IO
1295 */
1296static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work,
1297                     unsigned long dirtied_before)
1298{
1299        int moved;
1300        unsigned long time_expire_jif = dirtied_before;
1301
1302        assert_spin_locked(&wb->list_lock);
1303        list_splice_init(&wb->b_more_io, &wb->b_io);
1304        moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, dirtied_before);
1305        if (!work->for_sync)
1306                time_expire_jif = jiffies - dirtytime_expire_interval * HZ;
1307        moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io,
1308                                     time_expire_jif);
1309        if (moved)
1310                wb_io_lists_populated(wb);
1311        trace_writeback_queue_io(wb, work, dirtied_before, moved);
1312}
1313
1314static int write_inode(struct inode *inode, struct writeback_control *wbc)
1315{
1316        int ret;
1317
1318        if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) {
1319                trace_writeback_write_inode_start(inode, wbc);
1320                ret = inode->i_sb->s_op->write_inode(inode, wbc);
1321                trace_writeback_write_inode(inode, wbc);
1322                return ret;
1323        }
1324        return 0;
1325}
1326
1327/*
1328 * Wait for writeback on an inode to complete. Called with i_lock held.
1329 * Caller must make sure inode cannot go away when we drop i_lock.
1330 */
1331static void __inode_wait_for_writeback(struct inode *inode)
1332        __releases(inode->i_lock)
1333        __acquires(inode->i_lock)
1334{
1335        DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
1336        wait_queue_head_t *wqh;
1337
1338        wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
1339        while (inode->i_state & I_SYNC) {
1340                spin_unlock(&inode->i_lock);
1341                __wait_on_bit(wqh, &wq, bit_wait,
1342                              TASK_UNINTERRUPTIBLE);
1343                spin_lock(&inode->i_lock);
1344        }
1345}
1346
1347/*
1348 * Wait for writeback on an inode to complete. Caller must have inode pinned.
1349 */
1350void inode_wait_for_writeback(struct inode *inode)
1351{
1352        spin_lock(&inode->i_lock);
1353        __inode_wait_for_writeback(inode);
1354        spin_unlock(&inode->i_lock);
1355}
1356
1357/*
1358 * Sleep until I_SYNC is cleared. This function must be called with i_lock
1359 * held and drops it. It is aimed for callers not holding any inode reference
1360 * so once i_lock is dropped, inode can go away.
1361 */
1362static void inode_sleep_on_writeback(struct inode *inode)
1363        __releases(inode->i_lock)
1364{
1365        DEFINE_WAIT(wait);
1366        wait_queue_head_t *wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
1367        int sleep;
1368
1369        prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
1370        sleep = inode->i_state & I_SYNC;
1371        spin_unlock(&inode->i_lock);
1372        if (sleep)
1373                schedule();
1374        finish_wait(wqh, &wait);
1375}
1376
1377/*
1378 * Find proper writeback list for the inode depending on its current state and
1379 * possibly also change of its state while we were doing writeback.  Here we
1380 * handle things such as livelock prevention or fairness of writeback among
1381 * inodes. This function can be called only by flusher thread - noone else
1382 * processes all inodes in writeback lists and requeueing inodes behind flusher
1383 * thread's back can have unexpected consequences.
1384 */
1385static void requeue_inode(struct inode *inode, struct bdi_writeback *wb,
1386                          struct writeback_control *wbc)
1387{
1388        if (inode->i_state & I_FREEING)
1389                return;
1390
1391        /*
1392         * Sync livelock prevention. Each inode is tagged and synced in one
1393         * shot. If still dirty, it will be redirty_tail()'ed below.  Update
1394         * the dirty time to prevent enqueue and sync it again.
1395         */
1396        if ((inode->i_state & I_DIRTY) &&
1397            (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages))
1398                inode->dirtied_when = jiffies;
1399
1400        if (wbc->pages_skipped) {
1401                /*
1402                 * writeback is not making progress due to locked
1403                 * buffers. Skip this inode for now.
1404                 */
1405                redirty_tail_locked(inode, wb);
1406                return;
1407        }
1408
1409        if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) {
1410                /*
1411                 * We didn't write back all the pages.  nfs_writepages()
1412                 * sometimes bales out without doing anything.
1413                 */
1414                if (wbc->nr_to_write <= 0) {
1415                        /* Slice used up. Queue for next turn. */
1416                        requeue_io(inode, wb);
1417                } else {
1418                        /*
1419                         * Writeback blocked by something other than
1420                         * congestion. Delay the inode for some time to
1421                         * avoid spinning on the CPU (100% iowait)
1422                         * retrying writeback of the dirty page/inode
1423                         * that cannot be performed immediately.
1424                         */
1425                        redirty_tail_locked(inode, wb);
1426                }
1427        } else if (inode->i_state & I_DIRTY) {
1428                /*
1429                 * Filesystems can dirty the inode during writeback operations,
1430                 * such as delayed allocation during submission or metadata
1431                 * updates after data IO completion.
1432                 */
1433                redirty_tail_locked(inode, wb);
1434        } else if (inode->i_state & I_DIRTY_TIME) {
1435                inode->dirtied_when = jiffies;
1436                inode_io_list_move_locked(inode, wb, &wb->b_dirty_time);
1437                inode->i_state &= ~I_SYNC_QUEUED;
1438        } else {
1439                /* The inode is clean. Remove from writeback lists. */
1440                inode_io_list_del_locked(inode, wb);
1441        }
1442}
1443
1444/*
1445 * Write out an inode and its dirty pages. Do not update the writeback list
1446 * linkage. That is left to the caller. The caller is also responsible for
1447 * setting I_SYNC flag and calling inode_sync_complete() to clear it.
1448 */
1449static int
1450__writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
1451{
1452        struct address_space *mapping = inode->i_mapping;
1453        long nr_to_write = wbc->nr_to_write;
1454        unsigned dirty;
1455        int ret;
1456
1457        WARN_ON(!(inode->i_state & I_SYNC));
1458
1459        trace_writeback_single_inode_start(inode, wbc, nr_to_write);
1460
1461        ret = do_writepages(mapping, wbc);
1462
1463        /*
1464         * Make sure to wait on the data before writing out the metadata.
1465         * This is important for filesystems that modify metadata on data
1466         * I/O completion. We don't do it for sync(2) writeback because it has a
1467         * separate, external IO completion path and ->sync_fs for guaranteeing
1468         * inode metadata is written back correctly.
1469         */
1470        if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) {
1471                int err = filemap_fdatawait(mapping);
1472                if (ret == 0)
1473                        ret = err;
1474        }
1475
1476        /*
1477         * Some filesystems may redirty the inode during the writeback
1478         * due to delalloc, clear dirty metadata flags right before
1479         * write_inode()
1480         */
1481        spin_lock(&inode->i_lock);
1482
1483        dirty = inode->i_state & I_DIRTY;
1484        if ((inode->i_state & I_DIRTY_TIME) &&
1485            ((dirty & I_DIRTY_INODE) ||
1486             wbc->sync_mode == WB_SYNC_ALL || wbc->for_sync ||
1487             time_after(jiffies, inode->dirtied_time_when +
1488                        dirtytime_expire_interval * HZ))) {
1489                dirty |= I_DIRTY_TIME;
1490                trace_writeback_lazytime(inode);
1491        }
1492        inode->i_state &= ~dirty;
1493
1494        /*
1495         * Paired with smp_mb() in __mark_inode_dirty().  This allows
1496         * __mark_inode_dirty() to test i_state without grabbing i_lock -
1497         * either they see the I_DIRTY bits cleared or we see the dirtied
1498         * inode.
1499         *
1500         * I_DIRTY_PAGES is always cleared together above even if @mapping
1501         * still has dirty pages.  The flag is reinstated after smp_mb() if
1502         * necessary.  This guarantees that either __mark_inode_dirty()
1503         * sees clear I_DIRTY_PAGES or we see PAGECACHE_TAG_DIRTY.
1504         */
1505        smp_mb();
1506
1507        if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
1508                inode->i_state |= I_DIRTY_PAGES;
1509
1510        spin_unlock(&inode->i_lock);
1511
1512        if (dirty & I_DIRTY_TIME)
1513                mark_inode_dirty_sync(inode);
1514        /* Don't write the inode if only I_DIRTY_PAGES was set */
1515        if (dirty & ~I_DIRTY_PAGES) {
1516                int err = write_inode(inode, wbc);
1517                if (ret == 0)
1518                        ret = err;
1519        }
1520        trace_writeback_single_inode(inode, wbc, nr_to_write);
1521        return ret;
1522}
1523
1524/*
1525 * Write out an inode's dirty pages. Either the caller has an active reference
1526 * on the inode or the inode has I_WILL_FREE set.
1527 *
1528 * This function is designed to be called for writing back one inode which
1529 * we go e.g. from filesystem. Flusher thread uses __writeback_single_inode()
1530 * and does more profound writeback list handling in writeback_sb_inodes().
1531 */
1532static int writeback_single_inode(struct inode *inode,
1533                                  struct writeback_control *wbc)
1534{
1535        struct bdi_writeback *wb;
1536        int ret = 0;
1537
1538        spin_lock(&inode->i_lock);
1539        if (!atomic_read(&inode->i_count))
1540                WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
1541        else
1542                WARN_ON(inode->i_state & I_WILL_FREE);
1543
1544        if (inode->i_state & I_SYNC) {
1545                if (wbc->sync_mode != WB_SYNC_ALL)
1546                        goto out;
1547                /*
1548                 * It's a data-integrity sync. We must wait. Since callers hold
1549                 * inode reference or inode has I_WILL_FREE set, it cannot go
1550                 * away under us.
1551                 */
1552                __inode_wait_for_writeback(inode);
1553        }
1554        WARN_ON(inode->i_state & I_SYNC);
1555        /*
1556         * Skip inode if it is clean and we have no outstanding writeback in
1557         * WB_SYNC_ALL mode. We don't want to mess with writeback lists in this
1558         * function since flusher thread may be doing for example sync in
1559         * parallel and if we move the inode, it could get skipped. So here we
1560         * make sure inode is on some writeback list and leave it there unless
1561         * we have completely cleaned the inode.
1562         */
1563        if (!(inode->i_state & I_DIRTY_ALL) &&
1564            (wbc->sync_mode != WB_SYNC_ALL ||
1565             !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)))
1566                goto out;
1567        inode->i_state |= I_SYNC;
1568        wbc_attach_and_unlock_inode(wbc, inode);
1569
1570        ret = __writeback_single_inode(inode, wbc);
1571
1572        wbc_detach_inode(wbc);
1573
1574        wb = inode_to_wb_and_lock_list(inode);
1575        spin_lock(&inode->i_lock);
1576        /*
1577         * If inode is clean, remove it from writeback lists. Otherwise don't
1578         * touch it. See comment above for explanation.
1579         */
1580        if (!(inode->i_state & I_DIRTY_ALL))
1581                inode_io_list_del_locked(inode, wb);
1582        spin_unlock(&wb->list_lock);
1583        inode_sync_complete(inode);
1584out:
1585        spin_unlock(&inode->i_lock);
1586        return ret;
1587}
1588
1589static long writeback_chunk_size(struct bdi_writeback *wb,
1590                                 struct wb_writeback_work *work)
1591{
1592        long pages;
1593
1594        /*
1595         * WB_SYNC_ALL mode does livelock avoidance by syncing dirty
1596         * inodes/pages in one big loop. Setting wbc.nr_to_write=LONG_MAX
1597         * here avoids calling into writeback_inodes_wb() more than once.
1598         *
1599         * The intended call sequence for WB_SYNC_ALL writeback is:
1600         *
1601         *      wb_writeback()
1602         *          writeback_sb_inodes()       <== called only once
1603         *              write_cache_pages()     <== called once for each inode
1604         *                   (quickly) tag currently dirty pages
1605         *                   (maybe slowly) sync all tagged pages
1606         */
1607        if (work->sync_mode == WB_SYNC_ALL || work->tagged_writepages)
1608                pages = LONG_MAX;
1609        else {
1610                pages = min(wb->avg_write_bandwidth / 2,
1611                            global_wb_domain.dirty_limit / DIRTY_SCOPE);
1612                pages = min(pages, work->nr_pages);
1613                pages = round_down(pages + MIN_WRITEBACK_PAGES,
1614                                   MIN_WRITEBACK_PAGES);
1615        }
1616
1617        return pages;
1618}
1619
1620/*
1621 * Write a portion of b_io inodes which belong to @sb.
1622 *
1623 * Return the number of pages and/or inodes written.
1624 *
1625 * NOTE! This is called with wb->list_lock held, and will
1626 * unlock and relock that for each inode it ends up doing
1627 * IO for.
1628 */
1629static long writeback_sb_inodes(struct super_block *sb,
1630                                struct bdi_writeback *wb,
1631                                struct wb_writeback_work *work)
1632{
1633        struct writeback_control wbc = {
1634                .sync_mode              = work->sync_mode,
1635                .tagged_writepages      = work->tagged_writepages,
1636                .for_kupdate            = work->for_kupdate,
1637                .for_background         = work->for_background,
1638                .for_sync               = work->for_sync,
1639                .range_cyclic           = work->range_cyclic,
1640                .range_start            = 0,
1641                .range_end              = LLONG_MAX,
1642        };
1643        unsigned long start_time = jiffies;
1644        long write_chunk;
1645        long wrote = 0;  /* count both pages and inodes */
1646
1647        while (!list_empty(&wb->b_io)) {
1648                struct inode *inode = wb_inode(wb->b_io.prev);
1649                struct bdi_writeback *tmp_wb;
1650
1651                if (inode->i_sb != sb) {
1652                        if (work->sb) {
1653                                /*
1654                                 * We only want to write back data for this
1655                                 * superblock, move all inodes not belonging
1656                                 * to it back onto the dirty list.
1657                                 */
1658                                redirty_tail(inode, wb);
1659                                continue;
1660                        }
1661
1662                        /*
1663                         * The inode belongs to a different superblock.
1664                         * Bounce back to the caller to unpin this and
1665                         * pin the next superblock.
1666                         */
1667                        break;
1668                }
1669
1670                /*
1671                 * Don't bother with new inodes or inodes being freed, first
1672                 * kind does not need periodic writeout yet, and for the latter
1673                 * kind writeout is handled by the freer.
1674                 */
1675                spin_lock(&inode->i_lock);
1676                if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
1677                        redirty_tail_locked(inode, wb);
1678                        spin_unlock(&inode->i_lock);
1679                        continue;
1680                }
1681                if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) {
1682                        /*
1683                         * If this inode is locked for writeback and we are not
1684                         * doing writeback-for-data-integrity, move it to
1685                         * b_more_io so that writeback can proceed with the
1686                         * other inodes on s_io.
1687                         *
1688                         * We'll have another go at writing back this inode
1689                         * when we completed a full scan of b_io.
1690                         */
1691                        spin_unlock(&inode->i_lock);
1692                        requeue_io(inode, wb);
1693                        trace_writeback_sb_inodes_requeue(inode);
1694                        continue;
1695                }
1696                spin_unlock(&wb->list_lock);
1697
1698                /*
1699                 * We already requeued the inode if it had I_SYNC set and we
1700                 * are doing WB_SYNC_NONE writeback. So this catches only the
1701                 * WB_SYNC_ALL case.
1702                 */
1703                if (inode->i_state & I_SYNC) {
1704                        /* Wait for I_SYNC. This function drops i_lock... */
1705                        inode_sleep_on_writeback(inode);
1706                        /* Inode may be gone, start again */
1707                        spin_lock(&wb->list_lock);
1708                        continue;
1709                }
1710                inode->i_state |= I_SYNC;
1711                wbc_attach_and_unlock_inode(&wbc, inode);
1712
1713                write_chunk = writeback_chunk_size(wb, work);
1714                wbc.nr_to_write = write_chunk;
1715                wbc.pages_skipped = 0;
1716
1717                /*
1718                 * We use I_SYNC to pin the inode in memory. While it is set
1719                 * evict_inode() will wait so the inode cannot be freed.
1720                 */
1721                __writeback_single_inode(inode, &wbc);
1722
1723                wbc_detach_inode(&wbc);
1724                work->nr_pages -= write_chunk - wbc.nr_to_write;
1725                wrote += write_chunk - wbc.nr_to_write;
1726
1727                if (need_resched()) {
1728                        /*
1729                         * We're trying to balance between building up a nice
1730                         * long list of IOs to improve our merge rate, and
1731                         * getting those IOs out quickly for anyone throttling
1732                         * in balance_dirty_pages().  cond_resched() doesn't
1733                         * unplug, so get our IOs out the door before we
1734                         * give up the CPU.
1735                         */
1736                        blk_flush_plug(current);
1737                        cond_resched();
1738                }
1739
1740                /*
1741                 * Requeue @inode if still dirty.  Be careful as @inode may
1742                 * have been switched to another wb in the meantime.
1743                 */
1744                tmp_wb = inode_to_wb_and_lock_list(inode);
1745                spin_lock(&inode->i_lock);
1746                if (!(inode->i_state & I_DIRTY_ALL))
1747                        wrote++;
1748                requeue_inode(inode, tmp_wb, &wbc);
1749                inode_sync_complete(inode);
1750                spin_unlock(&inode->i_lock);
1751
1752                if (unlikely(tmp_wb != wb)) {
1753                        spin_unlock(&tmp_wb->list_lock);
1754                        spin_lock(&wb->list_lock);
1755                }
1756
1757                /*
1758                 * bail out to wb_writeback() often enough to check
1759                 * background threshold and other termination conditions.
1760                 */
1761                if (wrote) {
1762                        if (time_is_before_jiffies(start_time + HZ / 10UL))
1763                                break;
1764                        if (work->nr_pages <= 0)
1765                                break;
1766                }
1767        }
1768        return wrote;
1769}
1770
1771static long __writeback_inodes_wb(struct bdi_writeback *wb,
1772                                  struct wb_writeback_work *work)
1773{
1774        unsigned long start_time = jiffies;
1775        long wrote = 0;
1776
1777        while (!list_empty(&wb->b_io)) {
1778                struct inode *inode = wb_inode(wb->b_io.prev);
1779                struct super_block *sb = inode->i_sb;
1780
1781                if (!trylock_super(sb)) {
1782                        /*
1783                         * trylock_super() may fail consistently due to
1784                         * s_umount being grabbed by someone else. Don't use
1785                         * requeue_io() to avoid busy retrying the inode/sb.
1786                         */
1787                        redirty_tail(inode, wb);
1788                        continue;
1789                }
1790                wrote += writeback_sb_inodes(sb, wb, work);
1791                up_read(&sb->s_umount);
1792
1793                /* refer to the same tests at the end of writeback_sb_inodes */
1794                if (wrote) {
1795                        if (time_is_before_jiffies(start_time + HZ / 10UL))
1796                                break;
1797                        if (work->nr_pages <= 0)
1798                                break;
1799                }
1800        }
1801        /* Leave any unwritten inodes on b_io */
1802        return wrote;
1803}
1804
1805static long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages,
1806                                enum wb_reason reason)
1807{
1808        struct wb_writeback_work work = {
1809                .nr_pages       = nr_pages,
1810                .sync_mode      = WB_SYNC_NONE,
1811                .range_cyclic   = 1,
1812                .reason         = reason,
1813        };
1814        struct blk_plug plug;
1815
1816        blk_start_plug(&plug);
1817        spin_lock(&wb->list_lock);
1818        if (list_empty(&wb->b_io))
1819                queue_io(wb, &work, jiffies);
1820        __writeback_inodes_wb(wb, &work);
1821        spin_unlock(&wb->list_lock);
1822        blk_finish_plug(&plug);
1823
1824        return nr_pages - work.nr_pages;
1825}
1826
1827/*
1828 * Explicit flushing or periodic writeback of "old" data.
1829 *
1830 * Define "old": the first time one of an inode's pages is dirtied, we mark the
1831 * dirtying-time in the inode's address_space.  So this periodic writeback code
1832 * just walks the superblock inode list, writing back any inodes which are
1833 * older than a specific point in time.
1834 *
1835 * Try to run once per dirty_writeback_interval.  But if a writeback event
1836 * takes longer than a dirty_writeback_interval interval, then leave a
1837 * one-second gap.
1838 *
1839 * dirtied_before takes precedence over nr_to_write.  So we'll only write back
1840 * all dirty pages if they are all attached to "old" mappings.
1841 */
1842static long wb_writeback(struct bdi_writeback *wb,
1843                         struct wb_writeback_work *work)
1844{
1845        unsigned long wb_start = jiffies;
1846        long nr_pages = work->nr_pages;
1847        unsigned long dirtied_before = jiffies;
1848        struct inode *inode;
1849        long progress;
1850        struct blk_plug plug;
1851
1852        blk_start_plug(&plug);
1853        spin_lock(&wb->list_lock);
1854        for (;;) {
1855                /*
1856                 * Stop writeback when nr_pages has been consumed
1857                 */
1858                if (work->nr_pages <= 0)
1859                        break;
1860
1861                /*
1862                 * Background writeout and kupdate-style writeback may
1863                 * run forever. Stop them if there is other work to do
1864                 * so that e.g. sync can proceed. They'll be restarted
1865                 * after the other works are all done.
1866                 */
1867                if ((work->for_background || work->for_kupdate) &&
1868                    !list_empty(&wb->work_list))
1869                        break;
1870
1871                /*
1872                 * For background writeout, stop when we are below the
1873                 * background dirty threshold
1874                 */
1875                if (work->for_background && !wb_over_bg_thresh(wb))
1876                        break;
1877
1878                /*
1879                 * Kupdate and background works are special and we want to
1880                 * include all inodes that need writing. Livelock avoidance is
1881                 * handled by these works yielding to any other work so we are
1882                 * safe.
1883                 */
1884                if (work->for_kupdate) {
1885                        dirtied_before = jiffies -
1886                                msecs_to_jiffies(dirty_expire_interval * 10);
1887                } else if (work->for_background)
1888                        dirtied_before = jiffies;
1889
1890                trace_writeback_start(wb, work);
1891                if (list_empty(&wb->b_io))
1892                        queue_io(wb, work, dirtied_before);
1893                if (work->sb)
1894                        progress = writeback_sb_inodes(work->sb, wb, work);
1895                else
1896                        progress = __writeback_inodes_wb(wb, work);
1897                trace_writeback_written(wb, work);
1898
1899                wb_update_bandwidth(wb, wb_start);
1900
1901                /*
1902                 * Did we write something? Try for more
1903                 *
1904                 * Dirty inodes are moved to b_io for writeback in batches.
1905                 * The completion of the current batch does not necessarily
1906                 * mean the overall work is done. So we keep looping as long
1907                 * as made some progress on cleaning pages or inodes.
1908                 */
1909                if (progress)
1910                        continue;
1911                /*
1912                 * No more inodes for IO, bail
1913                 */
1914                if (list_empty(&wb->b_more_io))
1915                        break;
1916                /*
1917                 * Nothing written. Wait for some inode to
1918                 * become available for writeback. Otherwise
1919                 * we'll just busyloop.
1920                 */
1921                trace_writeback_wait(wb, work);
1922                inode = wb_inode(wb->b_more_io.prev);
1923                spin_lock(&inode->i_lock);
1924                spin_unlock(&wb->list_lock);
1925                /* This function drops i_lock... */
1926                inode_sleep_on_writeback(inode);
1927                spin_lock(&wb->list_lock);
1928        }
1929        spin_unlock(&wb->list_lock);
1930        blk_finish_plug(&plug);
1931
1932        return nr_pages - work->nr_pages;
1933}
1934
1935/*
1936 * Return the next wb_writeback_work struct that hasn't been processed yet.
1937 */
1938static struct wb_writeback_work *get_next_work_item(struct bdi_writeback *wb)
1939{
1940        struct wb_writeback_work *work = NULL;
1941
1942        spin_lock_bh(&wb->work_lock);
1943        if (!list_empty(&wb->work_list)) {
1944                work = list_entry(wb->work_list.next,
1945                                  struct wb_writeback_work, list);
1946                list_del_init(&work->list);
1947        }
1948        spin_unlock_bh(&wb->work_lock);
1949        return work;
1950}
1951
1952static long wb_check_background_flush(struct bdi_writeback *wb)
1953{
1954        if (wb_over_bg_thresh(wb)) {
1955
1956                struct wb_writeback_work work = {
1957                        .nr_pages       = LONG_MAX,
1958                        .sync_mode      = WB_SYNC_NONE,
1959                        .for_background = 1,
1960                        .range_cyclic   = 1,
1961                        .reason         = WB_REASON_BACKGROUND,
1962                };
1963
1964                return wb_writeback(wb, &work);
1965        }
1966
1967        return 0;
1968}
1969
1970static long wb_check_old_data_flush(struct bdi_writeback *wb)
1971{
1972        unsigned long expired;
1973        long nr_pages;
1974
1975        /*
1976         * When set to zero, disable periodic writeback
1977         */
1978        if (!dirty_writeback_interval)
1979                return 0;
1980
1981        expired = wb->last_old_flush +
1982                        msecs_to_jiffies(dirty_writeback_interval * 10);
1983        if (time_before(jiffies, expired))
1984                return 0;
1985
1986        wb->last_old_flush = jiffies;
1987        nr_pages = get_nr_dirty_pages();
1988
1989        if (nr_pages) {
1990                struct wb_writeback_work work = {
1991                        .nr_pages       = nr_pages,
1992                        .sync_mode      = WB_SYNC_NONE,
1993                        .for_kupdate    = 1,
1994                        .range_cyclic   = 1,
1995                        .reason         = WB_REASON_PERIODIC,
1996                };
1997
1998                return wb_writeback(wb, &work);
1999        }
2000
2001        return 0;
2002}
2003
2004static long wb_check_start_all(struct bdi_writeback *wb)
2005{
2006        long nr_pages;
2007
2008        if (!test_bit(WB_start_all, &wb->state))
2009                return 0;
2010
2011        nr_pages = get_nr_dirty_pages();
2012        if (nr_pages) {
2013                struct wb_writeback_work work = {
2014                        .nr_pages       = wb_split_bdi_pages(wb, nr_pages),
2015                        .sync_mode      = WB_SYNC_NONE,
2016                        .range_cyclic   = 1,
2017                        .reason         = wb->start_all_reason,
2018                };
2019
2020                nr_pages = wb_writeback(wb, &work);
2021        }
2022
2023        clear_bit(WB_start_all, &wb->state);
2024        return nr_pages;
2025}
2026
2027
2028/*
2029 * Retrieve work items and do the writeback they describe
2030 */
2031static long wb_do_writeback(struct bdi_writeback *wb)
2032{
2033        struct wb_writeback_work *work;
2034        long wrote = 0;
2035
2036        set_bit(WB_writeback_running, &wb->state);
2037        while ((work = get_next_work_item(wb)) != NULL) {
2038                trace_writeback_exec(wb, work);
2039                wrote += wb_writeback(wb, work);
2040                finish_writeback_work(wb, work);
2041        }
2042
2043        /*
2044         * Check for a flush-everything request
2045         */
2046        wrote += wb_check_start_all(wb);
2047
2048        /*
2049         * Check for periodic writeback, kupdated() style
2050         */
2051        wrote += wb_check_old_data_flush(wb);
2052        wrote += wb_check_background_flush(wb);
2053        clear_bit(WB_writeback_running, &wb->state);
2054
2055        return wrote;
2056}
2057
2058/*
2059 * Handle writeback of dirty data for the device backed by this bdi. Also
2060 * reschedules periodically and does kupdated style flushing.
2061 */
2062void wb_workfn(struct work_struct *work)
2063{
2064        struct bdi_writeback *wb = container_of(to_delayed_work(work),
2065                                                struct bdi_writeback, dwork);
2066        long pages_written;
2067
2068        set_worker_desc("flush-%s", bdi_dev_name(wb->bdi));
2069        current->flags |= PF_SWAPWRITE;
2070
2071        if (likely(!current_is_workqueue_rescuer() ||
2072                   !test_bit(WB_registered, &wb->state))) {
2073                /*
2074                 * The normal path.  Keep writing back @wb until its
2075                 * work_list is empty.  Note that this path is also taken
2076                 * if @wb is shutting down even when we're running off the
2077                 * rescuer as work_list needs to be drained.
2078                 */
2079                do {
2080                        pages_written = wb_do_writeback(wb);
2081                        trace_writeback_pages_written(pages_written);
2082                } while (!list_empty(&wb->work_list));
2083        } else {
2084                /*
2085                 * bdi_wq can't get enough workers and we're running off
2086                 * the emergency worker.  Don't hog it.  Hopefully, 1024 is
2087                 * enough for efficient IO.
2088                 */
2089                pages_written = writeback_inodes_wb(wb, 1024,
2090                                                    WB_REASON_FORKER_THREAD);
2091                trace_writeback_pages_written(pages_written);
2092        }
2093
2094        if (!list_empty(&wb->work_list))
2095                wb_wakeup(wb);
2096        else if (wb_has_dirty_io(wb) && dirty_writeback_interval)
2097                wb_wakeup_delayed(wb);
2098
2099        current->flags &= ~PF_SWAPWRITE;
2100}
2101
2102/*
2103 * Start writeback of `nr_pages' pages on this bdi. If `nr_pages' is zero,
2104 * write back the whole world.
2105 */
2106static void __wakeup_flusher_threads_bdi(struct backing_dev_info *bdi,
2107                                         enum wb_reason reason)
2108{
2109        struct bdi_writeback *wb;
2110
2111        if (!bdi_has_dirty_io(bdi))
2112                return;
2113
2114        list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)
2115                wb_start_writeback(wb, reason);
2116}
2117
2118void wakeup_flusher_threads_bdi(struct backing_dev_info *bdi,
2119                                enum wb_reason reason)
2120{
2121        rcu_read_lock();
2122        __wakeup_flusher_threads_bdi(bdi, reason);
2123        rcu_read_unlock();
2124}
2125
2126/*
2127 * Wakeup the flusher threads to start writeback of all currently dirty pages
2128 */
2129void wakeup_flusher_threads(enum wb_reason reason)
2130{
2131        struct backing_dev_info *bdi;
2132
2133        /*
2134         * If we are expecting writeback progress we must submit plugged IO.
2135         */
2136        if (blk_needs_flush_plug(current))
2137                blk_schedule_flush_plug(current);
2138
2139        rcu_read_lock();
2140        list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
2141                __wakeup_flusher_threads_bdi(bdi, reason);
2142        rcu_read_unlock();
2143}
2144
2145/*
2146 * Wake up bdi's periodically to make sure dirtytime inodes gets
2147 * written back periodically.  We deliberately do *not* check the
2148 * b_dirtytime list in wb_has_dirty_io(), since this would cause the
2149 * kernel to be constantly waking up once there are any dirtytime
2150 * inodes on the system.  So instead we define a separate delayed work
2151 * function which gets called much more rarely.  (By default, only
2152 * once every 12 hours.)
2153 *
2154 * If there is any other write activity going on in the file system,
2155 * this function won't be necessary.  But if the only thing that has
2156 * happened on the file system is a dirtytime inode caused by an atime
2157 * update, we need this infrastructure below to make sure that inode
2158 * eventually gets pushed out to disk.
2159 */
2160static void wakeup_dirtytime_writeback(struct work_struct *w);
2161static DECLARE_DELAYED_WORK(dirtytime_work, wakeup_dirtytime_writeback);
2162
2163static void wakeup_dirtytime_writeback(struct work_struct *w)
2164{
2165        struct backing_dev_info *bdi;
2166
2167        rcu_read_lock();
2168        list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
2169                struct bdi_writeback *wb;
2170
2171                list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)
2172                        if (!list_empty(&wb->b_dirty_time))
2173                                wb_wakeup(wb);
2174        }
2175        rcu_read_unlock();
2176        schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);
2177}
2178
2179static int __init start_dirtytime_writeback(void)
2180{
2181        schedule_delayed_work(&dirtytime_work, dirtytime_expire_interval * HZ);
2182        return 0;
2183}
2184__initcall(start_dirtytime_writeback);
2185
2186int dirtytime_interval_handler(struct ctl_table *table, int write,
2187                               void *buffer, size_t *lenp, loff_t *ppos)
2188{
2189        int ret;
2190
2191        ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
2192        if (ret == 0 && write)
2193                mod_delayed_work(system_wq, &dirtytime_work, 0);
2194        return ret;
2195}
2196
2197static noinline void block_dump___mark_inode_dirty(struct inode *inode)
2198{
2199        if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) {
2200                struct dentry *dentry;
2201                const char *name = "?";
2202
2203                dentry = d_find_alias(inode);
2204                if (dentry) {
2205                        spin_lock(&dentry->d_lock);
2206                        name = (const char *) dentry->d_name.name;
2207                }
2208                printk(KERN_DEBUG
2209                       "%s(%d): dirtied inode %lu (%s) on %s\n",
2210                       current->comm, task_pid_nr(current), inode->i_ino,
2211                       name, inode->i_sb->s_id);
2212                if (dentry) {
2213                        spin_unlock(&dentry->d_lock);
2214                        dput(dentry);
2215                }
2216        }
2217}
2218
2219/**
2220 * __mark_inode_dirty - internal function
2221 *
2222 * @inode: inode to mark
2223 * @flags: what kind of dirty (i.e. I_DIRTY_SYNC)
2224 *
2225 * Mark an inode as dirty. Callers should use mark_inode_dirty or
2226 * mark_inode_dirty_sync.
2227 *
2228 * Put the inode on the super block's dirty list.
2229 *
2230 * CAREFUL! We mark it dirty unconditionally, but move it onto the
2231 * dirty list only if it is hashed or if it refers to a blockdev.
2232 * If it was not hashed, it will never be added to the dirty list
2233 * even if it is later hashed, as it will have been marked dirty already.
2234 *
2235 * In short, make sure you hash any inodes _before_ you start marking
2236 * them dirty.
2237 *
2238 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
2239 * the block-special inode (/dev/hda1) itself.  And the ->dirtied_when field of
2240 * the kernel-internal blockdev inode represents the dirtying time of the
2241 * blockdev's pages.  This is why for I_DIRTY_PAGES we always use
2242 * page->mapping->host, so the page-dirtying time is recorded in the internal
2243 * blockdev inode.
2244 */
2245void __mark_inode_dirty(struct inode *inode, int flags)
2246{
2247        struct super_block *sb = inode->i_sb;
2248        int dirtytime;
2249
2250        trace_writeback_mark_inode_dirty(inode, flags);
2251
2252        /*
2253         * Don't do this for I_DIRTY_PAGES - that doesn't actually
2254         * dirty the inode itself
2255         */
2256        if (flags & (I_DIRTY_INODE | I_DIRTY_TIME)) {
2257                trace_writeback_dirty_inode_start(inode, flags);
2258
2259                if (sb->s_op->dirty_inode)
2260                        sb->s_op->dirty_inode(inode, flags);
2261
2262                trace_writeback_dirty_inode(inode, flags);
2263        }
2264        if (flags & I_DIRTY_INODE)
2265                flags &= ~I_DIRTY_TIME;
2266        dirtytime = flags & I_DIRTY_TIME;
2267
2268        /*
2269         * Paired with smp_mb() in __writeback_single_inode() for the
2270         * following lockless i_state test.  See there for details.
2271         */
2272        smp_mb();
2273
2274        if (((inode->i_state & flags) == flags) ||
2275            (dirtytime && (inode->i_state & I_DIRTY_INODE)))
2276                return;
2277
2278        if (unlikely(block_dump))
2279                block_dump___mark_inode_dirty(inode);
2280
2281        spin_lock(&inode->i_lock);
2282        if (dirtytime && (inode->i_state & I_DIRTY_INODE))
2283                goto out_unlock_inode;
2284        if ((inode->i_state & flags) != flags) {
2285                const int was_dirty = inode->i_state & I_DIRTY;
2286
2287                inode_attach_wb(inode, NULL);
2288
2289                if (flags & I_DIRTY_INODE)
2290                        inode->i_state &= ~I_DIRTY_TIME;
2291                inode->i_state |= flags;
2292
2293                /*
2294                 * If the inode is queued for writeback by flush worker, just
2295                 * update its dirty state. Once the flush worker is done with
2296                 * the inode it will place it on the appropriate superblock
2297                 * list, based upon its state.
2298                 */
2299                if (inode->i_state & I_SYNC_QUEUED)
2300                        goto out_unlock_inode;
2301
2302                /*
2303                 * Only add valid (hashed) inodes to the superblock's
2304                 * dirty list.  Add blockdev inodes as well.
2305                 */
2306                if (!S_ISBLK(inode->i_mode)) {
2307                        if (inode_unhashed(inode))
2308                                goto out_unlock_inode;
2309                }
2310                if (inode->i_state & I_FREEING)
2311                        goto out_unlock_inode;
2312
2313                /*
2314                 * If the inode was already on b_dirty/b_io/b_more_io, don't
2315                 * reposition it (that would break b_dirty time-ordering).
2316                 */
2317                if (!was_dirty) {
2318                        struct bdi_writeback *wb;
2319                        struct list_head *dirty_list;
2320                        bool wakeup_bdi = false;
2321
2322                        wb = locked_inode_to_wb_and_lock_list(inode);
2323
2324                        WARN((wb->bdi->capabilities & BDI_CAP_WRITEBACK) &&
2325                             !test_bit(WB_registered, &wb->state),
2326                             "bdi-%s not registered\n", bdi_dev_name(wb->bdi));
2327
2328                        inode->dirtied_when = jiffies;
2329                        if (dirtytime)
2330                                inode->dirtied_time_when = jiffies;
2331
2332                        if (inode->i_state & I_DIRTY)
2333                                dirty_list = &wb->b_dirty;
2334                        else
2335                                dirty_list = &wb->b_dirty_time;
2336
2337                        wakeup_bdi = inode_io_list_move_locked(inode, wb,
2338                                                               dirty_list);
2339
2340                        spin_unlock(&wb->list_lock);
2341                        trace_writeback_dirty_inode_enqueue(inode);
2342
2343                        /*
2344                         * If this is the first dirty inode for this bdi,
2345                         * we have to wake-up the corresponding bdi thread
2346                         * to make sure background write-back happens
2347                         * later.
2348                         */
2349                        if (wakeup_bdi &&
2350                            (wb->bdi->capabilities & BDI_CAP_WRITEBACK))
2351                                wb_wakeup_delayed(wb);
2352                        return;
2353                }
2354        }
2355out_unlock_inode:
2356        spin_unlock(&inode->i_lock);
2357}
2358EXPORT_SYMBOL(__mark_inode_dirty);
2359
2360/*
2361 * The @s_sync_lock is used to serialise concurrent sync operations
2362 * to avoid lock contention problems with concurrent wait_sb_inodes() calls.
2363 * Concurrent callers will block on the s_sync_lock rather than doing contending
2364 * walks. The queueing maintains sync(2) required behaviour as all the IO that
2365 * has been issued up to the time this function is enter is guaranteed to be
2366 * completed by the time we have gained the lock and waited for all IO that is
2367 * in progress regardless of the order callers are granted the lock.
2368 */
2369static void wait_sb_inodes(struct super_block *sb)
2370{
2371        LIST_HEAD(sync_list);
2372
2373        /*
2374         * We need to be protected against the filesystem going from
2375         * r/o to r/w or vice versa.
2376         */
2377        WARN_ON(!rwsem_is_locked(&sb->s_umount));
2378
2379        mutex_lock(&sb->s_sync_lock);
2380
2381        /*
2382         * Splice the writeback list onto a temporary list to avoid waiting on
2383         * inodes that have started writeback after this point.
2384         *
2385         * Use rcu_read_lock() to keep the inodes around until we have a
2386         * reference. s_inode_wblist_lock protects sb->s_inodes_wb as well as
2387         * the local list because inodes can be dropped from either by writeback
2388         * completion.
2389         */
2390        rcu_read_lock();
2391        spin_lock_irq(&sb->s_inode_wblist_lock);
2392        list_splice_init(&sb->s_inodes_wb, &sync_list);
2393
2394        /*
2395         * Data integrity sync. Must wait for all pages under writeback, because
2396         * there may have been pages dirtied before our sync call, but which had
2397         * writeout started before we write it out.  In which case, the inode
2398         * may not be on the dirty list, but we still have to wait for that
2399         * writeout.
2400         */
2401        while (!list_empty(&sync_list)) {
2402                struct inode *inode = list_first_entry(&sync_list, struct inode,
2403                                                       i_wb_list);
2404                struct address_space *mapping = inode->i_mapping;
2405
2406                /*
2407                 * Move each inode back to the wb list before we drop the lock
2408                 * to preserve consistency between i_wb_list and the mapping
2409                 * writeback tag. Writeback completion is responsible to remove
2410                 * the inode from either list once the writeback tag is cleared.
2411                 */
2412                list_move_tail(&inode->i_wb_list, &sb->s_inodes_wb);
2413
2414                /*
2415                 * The mapping can appear untagged while still on-list since we
2416                 * do not have the mapping lock. Skip it here, wb completion
2417                 * will remove it.
2418                 */
2419                if (!mapping_tagged(mapping, PAGECACHE_TAG_WRITEBACK))
2420                        continue;
2421
2422                spin_unlock_irq(&sb->s_inode_wblist_lock);
2423
2424                spin_lock(&inode->i_lock);
2425                if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) {
2426                        spin_unlock(&inode->i_lock);
2427
2428                        spin_lock_irq(&sb->s_inode_wblist_lock);
2429                        continue;
2430                }
2431                __iget(inode);
2432                spin_unlock(&inode->i_lock);
2433                rcu_read_unlock();
2434
2435                /*
2436                 * We keep the error status of individual mapping so that
2437                 * applications can catch the writeback error using fsync(2).
2438                 * See filemap_fdatawait_keep_errors() for details.
2439                 */
2440                filemap_fdatawait_keep_errors(mapping);
2441
2442                cond_resched();
2443
2444                iput(inode);
2445
2446                rcu_read_lock();
2447                spin_lock_irq(&sb->s_inode_wblist_lock);
2448        }
2449        spin_unlock_irq(&sb->s_inode_wblist_lock);
2450        rcu_read_unlock();
2451        mutex_unlock(&sb->s_sync_lock);
2452}
2453
2454static void __writeback_inodes_sb_nr(struct super_block *sb, unsigned long nr,
2455                                     enum wb_reason reason, bool skip_if_busy)
2456{
2457        struct backing_dev_info *bdi = sb->s_bdi;
2458        DEFINE_WB_COMPLETION(done, bdi);
2459        struct wb_writeback_work work = {
2460                .sb                     = sb,
2461                .sync_mode              = WB_SYNC_NONE,
2462                .tagged_writepages      = 1,
2463                .done                   = &done,
2464                .nr_pages               = nr,
2465                .reason                 = reason,
2466        };
2467
2468        if (!bdi_has_dirty_io(bdi) || bdi == &noop_backing_dev_info)
2469                return;
2470        WARN_ON(!rwsem_is_locked(&sb->s_umount));
2471
2472        bdi_split_work_to_wbs(sb->s_bdi, &work, skip_if_busy);
2473        wb_wait_for_completion(&done);
2474}
2475
2476/**
2477 * writeback_inodes_sb_nr -     writeback dirty inodes from given super_block
2478 * @sb: the superblock
2479 * @nr: the number of pages to write
2480 * @reason: reason why some writeback work initiated
2481 *
2482 * Start writeback on some inodes on this super_block. No guarantees are made
2483 * on how many (if any) will be written, and this function does not wait
2484 * for IO completion of submitted IO.
2485 */
2486void writeback_inodes_sb_nr(struct super_block *sb,
2487                            unsigned long nr,
2488                            enum wb_reason reason)
2489{
2490        __writeback_inodes_sb_nr(sb, nr, reason, false);
2491}
2492EXPORT_SYMBOL(writeback_inodes_sb_nr);
2493
2494/**
2495 * writeback_inodes_sb  -       writeback dirty inodes from given super_block
2496 * @sb: the superblock
2497 * @reason: reason why some writeback work was initiated
2498 *
2499 * Start writeback on some inodes on this super_block. No guarantees are made
2500 * on how many (if any) will be written, and this function does not wait
2501 * for IO completion of submitted IO.
2502 */
2503void writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
2504{
2505        return writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason);
2506}
2507EXPORT_SYMBOL(writeback_inodes_sb);
2508
2509/**
2510 * try_to_writeback_inodes_sb - try to start writeback if none underway
2511 * @sb: the superblock
2512 * @reason: reason why some writeback work was initiated
2513 *
2514 * Invoke __writeback_inodes_sb_nr if no writeback is currently underway.
2515 */
2516void try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason)
2517{
2518        if (!down_read_trylock(&sb->s_umount))
2519                return;
2520
2521        __writeback_inodes_sb_nr(sb, get_nr_dirty_pages(), reason, true);
2522        up_read(&sb->s_umount);
2523}
2524EXPORT_SYMBOL(try_to_writeback_inodes_sb);
2525
2526/**
2527 * sync_inodes_sb       -       sync sb inode pages
2528 * @sb: the superblock
2529 *
2530 * This function writes and waits on any dirty inode belonging to this
2531 * super_block.
2532 */
2533void sync_inodes_sb(struct super_block *sb)
2534{
2535        struct backing_dev_info *bdi = sb->s_bdi;
2536        DEFINE_WB_COMPLETION(done, bdi);
2537        struct wb_writeback_work work = {
2538                .sb             = sb,
2539                .sync_mode      = WB_SYNC_ALL,
2540                .nr_pages       = LONG_MAX,
2541                .range_cyclic   = 0,
2542                .done           = &done,
2543                .reason         = WB_REASON_SYNC,
2544                .for_sync       = 1,
2545        };
2546
2547        /*
2548         * Can't skip on !bdi_has_dirty() because we should wait for !dirty
2549         * inodes under writeback and I_DIRTY_TIME inodes ignored by
2550         * bdi_has_dirty() need to be written out too.
2551         */
2552        if (bdi == &noop_backing_dev_info)
2553                return;
2554        WARN_ON(!rwsem_is_locked(&sb->s_umount));
2555
2556        /* protect against inode wb switch, see inode_switch_wbs_work_fn() */
2557        bdi_down_write_wb_switch_rwsem(bdi);
2558        bdi_split_work_to_wbs(bdi, &work, false);
2559        wb_wait_for_completion(&done);
2560        bdi_up_write_wb_switch_rwsem(bdi);
2561
2562        wait_sb_inodes(sb);
2563}
2564EXPORT_SYMBOL(sync_inodes_sb);
2565
2566/**
2567 * write_inode_now      -       write an inode to disk
2568 * @inode: inode to write to disk
2569 * @sync: whether the write should be synchronous or not
2570 *
2571 * This function commits an inode to disk immediately if it is dirty. This is
2572 * primarily needed by knfsd.
2573 *
2574 * The caller must either have a ref on the inode or must have set I_WILL_FREE.
2575 */
2576int write_inode_now(struct inode *inode, int sync)
2577{
2578        struct writeback_control wbc = {
2579                .nr_to_write = LONG_MAX,
2580                .sync_mode = sync ? WB_SYNC_ALL : WB_SYNC_NONE,
2581                .range_start = 0,
2582                .range_end = LLONG_MAX,
2583        };
2584
2585        if (!mapping_can_writeback(inode->i_mapping))
2586                wbc.nr_to_write = 0;
2587
2588        might_sleep();
2589        return writeback_single_inode(inode, &wbc);
2590}
2591EXPORT_SYMBOL(write_inode_now);
2592
2593/**
2594 * sync_inode - write an inode and its pages to disk.
2595 * @inode: the inode to sync
2596 * @wbc: controls the writeback mode
2597 *
2598 * sync_inode() will write an inode and its pages to disk.  It will also
2599 * correctly update the inode on its superblock's dirty inode lists and will
2600 * update inode->i_state.
2601 *
2602 * The caller must have a ref on the inode.
2603 */
2604int sync_inode(struct inode *inode, struct writeback_control *wbc)
2605{
2606        return writeback_single_inode(inode, wbc);
2607}
2608EXPORT_SYMBOL(sync_inode);
2609
2610/**
2611 * sync_inode_metadata - write an inode to disk
2612 * @inode: the inode to sync
2613 * @wait: wait for I/O to complete.
2614 *
2615 * Write an inode to disk and adjust its dirty state after completion.
2616 *
2617 * Note: only writes the actual inode, no associated data or other metadata.
2618 */
2619int sync_inode_metadata(struct inode *inode, int wait)
2620{
2621        struct writeback_control wbc = {
2622                .sync_mode = wait ? WB_SYNC_ALL : WB_SYNC_NONE,
2623                .nr_to_write = 0, /* metadata-only */
2624        };
2625
2626        return sync_inode(inode, &wbc);
2627}
2628EXPORT_SYMBOL(sync_inode_metadata);
2629