linux/block/bio.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (C) 2001 Jens Axboe <axboe@kernel.dk>
   4 */
   5#include <linux/mm.h>
   6#include <linux/swap.h>
   7#include <linux/bio.h>
   8#include <linux/blkdev.h>
   9#include <linux/uio.h>
  10#include <linux/iocontext.h>
  11#include <linux/slab.h>
  12#include <linux/init.h>
  13#include <linux/kernel.h>
  14#include <linux/export.h>
  15#include <linux/mempool.h>
  16#include <linux/workqueue.h>
  17#include <linux/cgroup.h>
  18#include <linux/blk-cgroup.h>
  19#include <linux/highmem.h>
  20
  21#include <trace/events/block.h>
  22#include "blk.h"
  23#include "blk-rq-qos.h"
  24
  25/*
  26 * Test patch to inline a certain number of bi_io_vec's inside the bio
  27 * itself, to shrink a bio data allocation from two mempool calls to one
  28 */
  29#define BIO_INLINE_VECS         4
  30
  31/*
  32 * if you change this list, also change bvec_alloc or things will
  33 * break badly! cannot be bigger than what you can fit into an
  34 * unsigned short
  35 */
  36#define BV(x, n) { .nr_vecs = x, .name = "biovec-"#n }
  37static struct biovec_slab bvec_slabs[BVEC_POOL_NR] __read_mostly = {
  38        BV(1, 1), BV(4, 4), BV(16, 16), BV(64, 64), BV(128, 128), BV(BIO_MAX_PAGES, max),
  39};
  40#undef BV
  41
  42/*
  43 * fs_bio_set is the bio_set containing bio and iovec memory pools used by
  44 * IO code that does not need private memory pools.
  45 */
  46struct bio_set fs_bio_set;
  47EXPORT_SYMBOL(fs_bio_set);
  48
  49/*
  50 * Our slab pool management
  51 */
  52struct bio_slab {
  53        struct kmem_cache *slab;
  54        unsigned int slab_ref;
  55        unsigned int slab_size;
  56        char name[8];
  57};
  58static DEFINE_MUTEX(bio_slab_lock);
  59static struct bio_slab *bio_slabs;
  60static unsigned int bio_slab_nr, bio_slab_max;
  61
  62static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)
  63{
  64        unsigned int sz = sizeof(struct bio) + extra_size;
  65        struct kmem_cache *slab = NULL;
  66        struct bio_slab *bslab, *new_bio_slabs;
  67        unsigned int new_bio_slab_max;
  68        unsigned int i, entry = -1;
  69
  70        mutex_lock(&bio_slab_lock);
  71
  72        i = 0;
  73        while (i < bio_slab_nr) {
  74                bslab = &bio_slabs[i];
  75
  76                if (!bslab->slab && entry == -1)
  77                        entry = i;
  78                else if (bslab->slab_size == sz) {
  79                        slab = bslab->slab;
  80                        bslab->slab_ref++;
  81                        break;
  82                }
  83                i++;
  84        }
  85
  86        if (slab)
  87                goto out_unlock;
  88
  89        if (bio_slab_nr == bio_slab_max && entry == -1) {
  90                new_bio_slab_max = bio_slab_max << 1;
  91                new_bio_slabs = krealloc(bio_slabs,
  92                                         new_bio_slab_max * sizeof(struct bio_slab),
  93                                         GFP_KERNEL);
  94                if (!new_bio_slabs)
  95                        goto out_unlock;
  96                bio_slab_max = new_bio_slab_max;
  97                bio_slabs = new_bio_slabs;
  98        }
  99        if (entry == -1)
 100                entry = bio_slab_nr++;
 101
 102        bslab = &bio_slabs[entry];
 103
 104        snprintf(bslab->name, sizeof(bslab->name), "bio-%d", entry);
 105        slab = kmem_cache_create(bslab->name, sz, ARCH_KMALLOC_MINALIGN,
 106                                 SLAB_HWCACHE_ALIGN, NULL);
 107        if (!slab)
 108                goto out_unlock;
 109
 110        bslab->slab = slab;
 111        bslab->slab_ref = 1;
 112        bslab->slab_size = sz;
 113out_unlock:
 114        mutex_unlock(&bio_slab_lock);
 115        return slab;
 116}
 117
 118static void bio_put_slab(struct bio_set *bs)
 119{
 120        struct bio_slab *bslab = NULL;
 121        unsigned int i;
 122
 123        mutex_lock(&bio_slab_lock);
 124
 125        for (i = 0; i < bio_slab_nr; i++) {
 126                if (bs->bio_slab == bio_slabs[i].slab) {
 127                        bslab = &bio_slabs[i];
 128                        break;
 129                }
 130        }
 131
 132        if (WARN(!bslab, KERN_ERR "bio: unable to find slab!\n"))
 133                goto out;
 134
 135        WARN_ON(!bslab->slab_ref);
 136
 137        if (--bslab->slab_ref)
 138                goto out;
 139
 140        kmem_cache_destroy(bslab->slab);
 141        bslab->slab = NULL;
 142
 143out:
 144        mutex_unlock(&bio_slab_lock);
 145}
 146
 147unsigned int bvec_nr_vecs(unsigned short idx)
 148{
 149        return bvec_slabs[--idx].nr_vecs;
 150}
 151
 152void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned int idx)
 153{
 154        if (!idx)
 155                return;
 156        idx--;
 157
 158        BIO_BUG_ON(idx >= BVEC_POOL_NR);
 159
 160        if (idx == BVEC_POOL_MAX) {
 161                mempool_free(bv, pool);
 162        } else {
 163                struct biovec_slab *bvs = bvec_slabs + idx;
 164
 165                kmem_cache_free(bvs->slab, bv);
 166        }
 167}
 168
 169struct bio_vec *bvec_alloc(gfp_t gfp_mask, int nr, unsigned long *idx,
 170                           mempool_t *pool)
 171{
 172        struct bio_vec *bvl;
 173
 174        /*
 175         * see comment near bvec_array define!
 176         */
 177        switch (nr) {
 178        case 1:
 179                *idx = 0;
 180                break;
 181        case 2 ... 4:
 182                *idx = 1;
 183                break;
 184        case 5 ... 16:
 185                *idx = 2;
 186                break;
 187        case 17 ... 64:
 188                *idx = 3;
 189                break;
 190        case 65 ... 128:
 191                *idx = 4;
 192                break;
 193        case 129 ... BIO_MAX_PAGES:
 194                *idx = 5;
 195                break;
 196        default:
 197                return NULL;
 198        }
 199
 200        /*
 201         * idx now points to the pool we want to allocate from. only the
 202         * 1-vec entry pool is mempool backed.
 203         */
 204        if (*idx == BVEC_POOL_MAX) {
 205fallback:
 206                bvl = mempool_alloc(pool, gfp_mask);
 207        } else {
 208                struct biovec_slab *bvs = bvec_slabs + *idx;
 209                gfp_t __gfp_mask = gfp_mask & ~(__GFP_DIRECT_RECLAIM | __GFP_IO);
 210
 211                /*
 212                 * Make this allocation restricted and don't dump info on
 213                 * allocation failures, since we'll fallback to the mempool
 214                 * in case of failure.
 215                 */
 216                __gfp_mask |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
 217
 218                /*
 219                 * Try a slab allocation. If this fails and __GFP_DIRECT_RECLAIM
 220                 * is set, retry with the 1-entry mempool
 221                 */
 222                bvl = kmem_cache_alloc(bvs->slab, __gfp_mask);
 223                if (unlikely(!bvl && (gfp_mask & __GFP_DIRECT_RECLAIM))) {
 224                        *idx = BVEC_POOL_MAX;
 225                        goto fallback;
 226                }
 227        }
 228
 229        (*idx)++;
 230        return bvl;
 231}
 232
 233void bio_uninit(struct bio *bio)
 234{
 235        bio_disassociate_blkg(bio);
 236}
 237EXPORT_SYMBOL(bio_uninit);
 238
 239static void bio_free(struct bio *bio)
 240{
 241        struct bio_set *bs = bio->bi_pool;
 242        void *p;
 243
 244        bio_uninit(bio);
 245
 246        if (bs) {
 247                bvec_free(&bs->bvec_pool, bio->bi_io_vec, BVEC_POOL_IDX(bio));
 248
 249                /*
 250                 * If we have front padding, adjust the bio pointer before freeing
 251                 */
 252                p = bio;
 253                p -= bs->front_pad;
 254
 255                mempool_free(p, &bs->bio_pool);
 256        } else {
 257                /* Bio was allocated by bio_kmalloc() */
 258                kfree(bio);
 259        }
 260}
 261
 262/*
 263 * Users of this function have their own bio allocation. Subsequently,
 264 * they must remember to pair any call to bio_init() with bio_uninit()
 265 * when IO has completed, or when the bio is released.
 266 */
 267void bio_init(struct bio *bio, struct bio_vec *table,
 268              unsigned short max_vecs)
 269{
 270        memset(bio, 0, sizeof(*bio));
 271        atomic_set(&bio->__bi_remaining, 1);
 272        atomic_set(&bio->__bi_cnt, 1);
 273
 274        bio->bi_io_vec = table;
 275        bio->bi_max_vecs = max_vecs;
 276}
 277EXPORT_SYMBOL(bio_init);
 278
 279/**
 280 * bio_reset - reinitialize a bio
 281 * @bio:        bio to reset
 282 *
 283 * Description:
 284 *   After calling bio_reset(), @bio will be in the same state as a freshly
 285 *   allocated bio returned bio bio_alloc_bioset() - the only fields that are
 286 *   preserved are the ones that are initialized by bio_alloc_bioset(). See
 287 *   comment in struct bio.
 288 */
 289void bio_reset(struct bio *bio)
 290{
 291        unsigned long flags = bio->bi_flags & (~0UL << BIO_RESET_BITS);
 292
 293        bio_uninit(bio);
 294
 295        memset(bio, 0, BIO_RESET_BYTES);
 296        bio->bi_flags = flags;
 297        atomic_set(&bio->__bi_remaining, 1);
 298}
 299EXPORT_SYMBOL(bio_reset);
 300
 301static struct bio *__bio_chain_endio(struct bio *bio)
 302{
 303        struct bio *parent = bio->bi_private;
 304
 305        if (!parent->bi_status)
 306                parent->bi_status = bio->bi_status;
 307        bio_put(bio);
 308        return parent;
 309}
 310
 311static void bio_chain_endio(struct bio *bio)
 312{
 313        bio_endio(__bio_chain_endio(bio));
 314}
 315
 316/**
 317 * bio_chain - chain bio completions
 318 * @bio: the target bio
 319 * @parent: the @bio's parent bio
 320 *
 321 * The caller won't have a bi_end_io called when @bio completes - instead,
 322 * @parent's bi_end_io won't be called until both @parent and @bio have
 323 * completed; the chained bio will also be freed when it completes.
 324 *
 325 * The caller must not set bi_private or bi_end_io in @bio.
 326 */
 327void bio_chain(struct bio *bio, struct bio *parent)
 328{
 329        BUG_ON(bio->bi_private || bio->bi_end_io);
 330
 331        bio->bi_private = parent;
 332        bio->bi_end_io  = bio_chain_endio;
 333        bio_inc_remaining(parent);
 334}
 335EXPORT_SYMBOL(bio_chain);
 336
 337static void bio_alloc_rescue(struct work_struct *work)
 338{
 339        struct bio_set *bs = container_of(work, struct bio_set, rescue_work);
 340        struct bio *bio;
 341
 342        while (1) {
 343                spin_lock(&bs->rescue_lock);
 344                bio = bio_list_pop(&bs->rescue_list);
 345                spin_unlock(&bs->rescue_lock);
 346
 347                if (!bio)
 348                        break;
 349
 350                generic_make_request(bio);
 351        }
 352}
 353
 354static void punt_bios_to_rescuer(struct bio_set *bs)
 355{
 356        struct bio_list punt, nopunt;
 357        struct bio *bio;
 358
 359        if (WARN_ON_ONCE(!bs->rescue_workqueue))
 360                return;
 361        /*
 362         * In order to guarantee forward progress we must punt only bios that
 363         * were allocated from this bio_set; otherwise, if there was a bio on
 364         * there for a stacking driver higher up in the stack, processing it
 365         * could require allocating bios from this bio_set, and doing that from
 366         * our own rescuer would be bad.
 367         *
 368         * Since bio lists are singly linked, pop them all instead of trying to
 369         * remove from the middle of the list:
 370         */
 371
 372        bio_list_init(&punt);
 373        bio_list_init(&nopunt);
 374
 375        while ((bio = bio_list_pop(&current->bio_list[0])))
 376                bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
 377        current->bio_list[0] = nopunt;
 378
 379        bio_list_init(&nopunt);
 380        while ((bio = bio_list_pop(&current->bio_list[1])))
 381                bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);
 382        current->bio_list[1] = nopunt;
 383
 384        spin_lock(&bs->rescue_lock);
 385        bio_list_merge(&bs->rescue_list, &punt);
 386        spin_unlock(&bs->rescue_lock);
 387
 388        queue_work(bs->rescue_workqueue, &bs->rescue_work);
 389}
 390
 391/**
 392 * bio_alloc_bioset - allocate a bio for I/O
 393 * @gfp_mask:   the GFP_* mask given to the slab allocator
 394 * @nr_iovecs:  number of iovecs to pre-allocate
 395 * @bs:         the bio_set to allocate from.
 396 *
 397 * Description:
 398 *   If @bs is NULL, uses kmalloc() to allocate the bio; else the allocation is
 399 *   backed by the @bs's mempool.
 400 *
 401 *   When @bs is not NULL, if %__GFP_DIRECT_RECLAIM is set then bio_alloc will
 402 *   always be able to allocate a bio. This is due to the mempool guarantees.
 403 *   To make this work, callers must never allocate more than 1 bio at a time
 404 *   from this pool. Callers that need to allocate more than 1 bio must always
 405 *   submit the previously allocated bio for IO before attempting to allocate
 406 *   a new one. Failure to do so can cause deadlocks under memory pressure.
 407 *
 408 *   Note that when running under generic_make_request() (i.e. any block
 409 *   driver), bios are not submitted until after you return - see the code in
 410 *   generic_make_request() that converts recursion into iteration, to prevent
 411 *   stack overflows.
 412 *
 413 *   This would normally mean allocating multiple bios under
 414 *   generic_make_request() would be susceptible to deadlocks, but we have
 415 *   deadlock avoidance code that resubmits any blocked bios from a rescuer
 416 *   thread.
 417 *
 418 *   However, we do not guarantee forward progress for allocations from other
 419 *   mempools. Doing multiple allocations from the same mempool under
 420 *   generic_make_request() should be avoided - instead, use bio_set's front_pad
 421 *   for per bio allocations.
 422 *
 423 *   RETURNS:
 424 *   Pointer to new bio on success, NULL on failure.
 425 */
 426struct bio *bio_alloc_bioset(gfp_t gfp_mask, unsigned int nr_iovecs,
 427                             struct bio_set *bs)
 428{
 429        gfp_t saved_gfp = gfp_mask;
 430        unsigned front_pad;
 431        unsigned inline_vecs;
 432        struct bio_vec *bvl = NULL;
 433        struct bio *bio;
 434        void *p;
 435
 436        if (!bs) {
 437                if (nr_iovecs > UIO_MAXIOV)
 438                        return NULL;
 439
 440                p = kmalloc(sizeof(struct bio) +
 441                            nr_iovecs * sizeof(struct bio_vec),
 442                            gfp_mask);
 443                front_pad = 0;
 444                inline_vecs = nr_iovecs;
 445        } else {
 446                /* should not use nobvec bioset for nr_iovecs > 0 */
 447                if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) &&
 448                                 nr_iovecs > 0))
 449                        return NULL;
 450                /*
 451                 * generic_make_request() converts recursion to iteration; this
 452                 * means if we're running beneath it, any bios we allocate and
 453                 * submit will not be submitted (and thus freed) until after we
 454                 * return.
 455                 *
 456                 * This exposes us to a potential deadlock if we allocate
 457                 * multiple bios from the same bio_set() while running
 458                 * underneath generic_make_request(). If we were to allocate
 459                 * multiple bios (say a stacking block driver that was splitting
 460                 * bios), we would deadlock if we exhausted the mempool's
 461                 * reserve.
 462                 *
 463                 * We solve this, and guarantee forward progress, with a rescuer
 464                 * workqueue per bio_set. If we go to allocate and there are
 465                 * bios on current->bio_list, we first try the allocation
 466                 * without __GFP_DIRECT_RECLAIM; if that fails, we punt those
 467                 * bios we would be blocking to the rescuer workqueue before
 468                 * we retry with the original gfp_flags.
 469                 */
 470
 471                if (current->bio_list &&
 472                    (!bio_list_empty(&current->bio_list[0]) ||
 473                     !bio_list_empty(&current->bio_list[1])) &&
 474                    bs->rescue_workqueue)
 475                        gfp_mask &= ~__GFP_DIRECT_RECLAIM;
 476
 477                p = mempool_alloc(&bs->bio_pool, gfp_mask);
 478                if (!p && gfp_mask != saved_gfp) {
 479                        punt_bios_to_rescuer(bs);
 480                        gfp_mask = saved_gfp;
 481                        p = mempool_alloc(&bs->bio_pool, gfp_mask);
 482                }
 483
 484                front_pad = bs->front_pad;
 485                inline_vecs = BIO_INLINE_VECS;
 486        }
 487
 488        if (unlikely(!p))
 489                return NULL;
 490
 491        bio = p + front_pad;
 492        bio_init(bio, NULL, 0);
 493
 494        if (nr_iovecs > inline_vecs) {
 495                unsigned long idx = 0;
 496
 497                bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
 498                if (!bvl && gfp_mask != saved_gfp) {
 499                        punt_bios_to_rescuer(bs);
 500                        gfp_mask = saved_gfp;
 501                        bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, &bs->bvec_pool);
 502                }
 503
 504                if (unlikely(!bvl))
 505                        goto err_free;
 506
 507                bio->bi_flags |= idx << BVEC_POOL_OFFSET;
 508        } else if (nr_iovecs) {
 509                bvl = bio->bi_inline_vecs;
 510        }
 511
 512        bio->bi_pool = bs;
 513        bio->bi_max_vecs = nr_iovecs;
 514        bio->bi_io_vec = bvl;
 515        return bio;
 516
 517err_free:
 518        mempool_free(p, &bs->bio_pool);
 519        return NULL;
 520}
 521EXPORT_SYMBOL(bio_alloc_bioset);
 522
 523void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
 524{
 525        unsigned long flags;
 526        struct bio_vec bv;
 527        struct bvec_iter iter;
 528
 529        __bio_for_each_segment(bv, bio, iter, start) {
 530                char *data = bvec_kmap_irq(&bv, &flags);
 531                memset(data, 0, bv.bv_len);
 532                flush_dcache_page(bv.bv_page);
 533                bvec_kunmap_irq(data, &flags);
 534        }
 535}
 536EXPORT_SYMBOL(zero_fill_bio_iter);
 537
 538/**
 539 * bio_put - release a reference to a bio
 540 * @bio:   bio to release reference to
 541 *
 542 * Description:
 543 *   Put a reference to a &struct bio, either one you have gotten with
 544 *   bio_alloc, bio_get or bio_clone_*. The last put of a bio will free it.
 545 **/
 546void bio_put(struct bio *bio)
 547{
 548        if (!bio_flagged(bio, BIO_REFFED))
 549                bio_free(bio);
 550        else {
 551                BIO_BUG_ON(!atomic_read(&bio->__bi_cnt));
 552
 553                /*
 554                 * last put frees it
 555                 */
 556                if (atomic_dec_and_test(&bio->__bi_cnt))
 557                        bio_free(bio);
 558        }
 559}
 560EXPORT_SYMBOL(bio_put);
 561
 562/**
 563 *      __bio_clone_fast - clone a bio that shares the original bio's biovec
 564 *      @bio: destination bio
 565 *      @bio_src: bio to clone
 566 *
 567 *      Clone a &bio. Caller will own the returned bio, but not
 568 *      the actual data it points to. Reference count of returned
 569 *      bio will be one.
 570 *
 571 *      Caller must ensure that @bio_src is not freed before @bio.
 572 */
 573void __bio_clone_fast(struct bio *bio, struct bio *bio_src)
 574{
 575        BUG_ON(bio->bi_pool && BVEC_POOL_IDX(bio));
 576
 577        /*
 578         * most users will be overriding ->bi_disk with a new target,
 579         * so we don't set nor calculate new physical/hw segment counts here
 580         */
 581        bio->bi_disk = bio_src->bi_disk;
 582        bio->bi_partno = bio_src->bi_partno;
 583        bio_set_flag(bio, BIO_CLONED);
 584        if (bio_flagged(bio_src, BIO_THROTTLED))
 585                bio_set_flag(bio, BIO_THROTTLED);
 586        bio->bi_opf = bio_src->bi_opf;
 587        bio->bi_ioprio = bio_src->bi_ioprio;
 588        bio->bi_write_hint = bio_src->bi_write_hint;
 589        bio->bi_iter = bio_src->bi_iter;
 590        bio->bi_io_vec = bio_src->bi_io_vec;
 591
 592        bio_clone_blkg_association(bio, bio_src);
 593        blkcg_bio_issue_init(bio);
 594}
 595EXPORT_SYMBOL(__bio_clone_fast);
 596
 597/**
 598 *      bio_clone_fast - clone a bio that shares the original bio's biovec
 599 *      @bio: bio to clone
 600 *      @gfp_mask: allocation priority
 601 *      @bs: bio_set to allocate from
 602 *
 603 *      Like __bio_clone_fast, only also allocates the returned bio
 604 */
 605struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
 606{
 607        struct bio *b;
 608
 609        b = bio_alloc_bioset(gfp_mask, 0, bs);
 610        if (!b)
 611                return NULL;
 612
 613        __bio_clone_fast(b, bio);
 614
 615        if (bio_integrity(bio)) {
 616                int ret;
 617
 618                ret = bio_integrity_clone(b, bio, gfp_mask);
 619
 620                if (ret < 0) {
 621                        bio_put(b);
 622                        return NULL;
 623                }
 624        }
 625
 626        return b;
 627}
 628EXPORT_SYMBOL(bio_clone_fast);
 629
 630static inline bool page_is_mergeable(const struct bio_vec *bv,
 631                struct page *page, unsigned int len, unsigned int off,
 632                bool *same_page)
 633{
 634        phys_addr_t vec_end_addr = page_to_phys(bv->bv_page) +
 635                bv->bv_offset + bv->bv_len - 1;
 636        phys_addr_t page_addr = page_to_phys(page);
 637
 638        if (vec_end_addr + 1 != page_addr + off)
 639                return false;
 640        if (xen_domain() && !xen_biovec_phys_mergeable(bv, page))
 641                return false;
 642
 643        *same_page = ((vec_end_addr & PAGE_MASK) == page_addr);
 644        if (!*same_page && pfn_to_page(PFN_DOWN(vec_end_addr)) + 1 != page)
 645                return false;
 646        return true;
 647}
 648
 649/*
 650 * Check if the @page can be added to the current segment(@bv), and make
 651 * sure to call it only if page_is_mergeable(@bv, @page) is true
 652 */
 653static bool can_add_page_to_seg(struct request_queue *q,
 654                struct bio_vec *bv, struct page *page, unsigned len,
 655                unsigned offset)
 656{
 657        unsigned long mask = queue_segment_boundary(q);
 658        phys_addr_t addr1 = page_to_phys(bv->bv_page) + bv->bv_offset;
 659        phys_addr_t addr2 = page_to_phys(page) + offset + len - 1;
 660
 661        if ((addr1 | mask) != (addr2 | mask))
 662                return false;
 663
 664        if (bv->bv_len + len > queue_max_segment_size(q))
 665                return false;
 666
 667        return true;
 668}
 669
 670/**
 671 *      __bio_add_pc_page       - attempt to add page to passthrough bio
 672 *      @q: the target queue
 673 *      @bio: destination bio
 674 *      @page: page to add
 675 *      @len: vec entry length
 676 *      @offset: vec entry offset
 677 *      @put_same_page: put the page if it is same with last added page
 678 *
 679 *      Attempt to add a page to the bio_vec maplist. This can fail for a
 680 *      number of reasons, such as the bio being full or target block device
 681 *      limitations. The target block device must allow bio's up to PAGE_SIZE,
 682 *      so it is always possible to add a single page to an empty bio.
 683 *
 684 *      This should only be used by passthrough bios.
 685 */
 686static int __bio_add_pc_page(struct request_queue *q, struct bio *bio,
 687                struct page *page, unsigned int len, unsigned int offset,
 688                bool put_same_page)
 689{
 690        struct bio_vec *bvec;
 691        bool same_page = false;
 692
 693        /*
 694         * cloned bio must not modify vec list
 695         */
 696        if (unlikely(bio_flagged(bio, BIO_CLONED)))
 697                return 0;
 698
 699        if (((bio->bi_iter.bi_size + len) >> 9) > queue_max_hw_sectors(q))
 700                return 0;
 701
 702        if (bio->bi_vcnt > 0) {
 703                bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
 704
 705                if (page == bvec->bv_page &&
 706                    offset == bvec->bv_offset + bvec->bv_len) {
 707                        if (put_same_page)
 708                                put_page(page);
 709                        bvec->bv_len += len;
 710                        goto done;
 711                }
 712
 713                /*
 714                 * If the queue doesn't support SG gaps and adding this
 715                 * offset would create a gap, disallow it.
 716                 */
 717                if (bvec_gap_to_prev(q, bvec, offset))
 718                        return 0;
 719
 720                if (page_is_mergeable(bvec, page, len, offset, &same_page) &&
 721                    can_add_page_to_seg(q, bvec, page, len, offset)) {
 722                        bvec->bv_len += len;
 723                        goto done;
 724                }
 725        }
 726
 727        if (bio_full(bio, len))
 728                return 0;
 729
 730        if (bio->bi_vcnt >= queue_max_segments(q))
 731                return 0;
 732
 733        bvec = &bio->bi_io_vec[bio->bi_vcnt];
 734        bvec->bv_page = page;
 735        bvec->bv_len = len;
 736        bvec->bv_offset = offset;
 737        bio->bi_vcnt++;
 738 done:
 739        bio->bi_iter.bi_size += len;
 740        return len;
 741}
 742
 743int bio_add_pc_page(struct request_queue *q, struct bio *bio,
 744                struct page *page, unsigned int len, unsigned int offset)
 745{
 746        return __bio_add_pc_page(q, bio, page, len, offset, false);
 747}
 748EXPORT_SYMBOL(bio_add_pc_page);
 749
 750/**
 751 * __bio_try_merge_page - try appending data to an existing bvec.
 752 * @bio: destination bio
 753 * @page: start page to add
 754 * @len: length of the data to add
 755 * @off: offset of the data relative to @page
 756 * @same_page: return if the segment has been merged inside the same page
 757 *
 758 * Try to add the data at @page + @off to the last bvec of @bio.  This is a
 759 * a useful optimisation for file systems with a block size smaller than the
 760 * page size.
 761 *
 762 * Warn if (@len, @off) crosses pages in case that @same_page is true.
 763 *
 764 * Return %true on success or %false on failure.
 765 */
 766bool __bio_try_merge_page(struct bio *bio, struct page *page,
 767                unsigned int len, unsigned int off, bool *same_page)
 768{
 769        if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
 770                return false;
 771
 772        if (bio->bi_vcnt > 0) {
 773                struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1];
 774
 775                if (page_is_mergeable(bv, page, len, off, same_page)) {
 776                        bv->bv_len += len;
 777                        bio->bi_iter.bi_size += len;
 778                        return true;
 779                }
 780        }
 781        return false;
 782}
 783EXPORT_SYMBOL_GPL(__bio_try_merge_page);
 784
 785/**
 786 * __bio_add_page - add page(s) to a bio in a new segment
 787 * @bio: destination bio
 788 * @page: start page to add
 789 * @len: length of the data to add, may cross pages
 790 * @off: offset of the data relative to @page, may cross pages
 791 *
 792 * Add the data at @page + @off to @bio as a new bvec.  The caller must ensure
 793 * that @bio has space for another bvec.
 794 */
 795void __bio_add_page(struct bio *bio, struct page *page,
 796                unsigned int len, unsigned int off)
 797{
 798        struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
 799
 800        WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
 801        WARN_ON_ONCE(bio_full(bio, len));
 802
 803        bv->bv_page = page;
 804        bv->bv_offset = off;
 805        bv->bv_len = len;
 806
 807        bio->bi_iter.bi_size += len;
 808        bio->bi_vcnt++;
 809}
 810EXPORT_SYMBOL_GPL(__bio_add_page);
 811
 812/**
 813 *      bio_add_page    -       attempt to add page(s) to bio
 814 *      @bio: destination bio
 815 *      @page: start page to add
 816 *      @len: vec entry length, may cross pages
 817 *      @offset: vec entry offset relative to @page, may cross pages
 818 *
 819 *      Attempt to add page(s) to the bio_vec maplist. This will only fail
 820 *      if either bio->bi_vcnt == bio->bi_max_vecs or it's a cloned bio.
 821 */
 822int bio_add_page(struct bio *bio, struct page *page,
 823                 unsigned int len, unsigned int offset)
 824{
 825        bool same_page = false;
 826
 827        if (!__bio_try_merge_page(bio, page, len, offset, &same_page)) {
 828                if (bio_full(bio, len))
 829                        return 0;
 830                __bio_add_page(bio, page, len, offset);
 831        }
 832        return len;
 833}
 834EXPORT_SYMBOL(bio_add_page);
 835
 836void bio_release_pages(struct bio *bio, bool mark_dirty)
 837{
 838        struct bvec_iter_all iter_all;
 839        struct bio_vec *bvec;
 840
 841        if (bio_flagged(bio, BIO_NO_PAGE_REF))
 842                return;
 843
 844        bio_for_each_segment_all(bvec, bio, iter_all) {
 845                if (mark_dirty && !PageCompound(bvec->bv_page))
 846                        set_page_dirty_lock(bvec->bv_page);
 847                put_page(bvec->bv_page);
 848        }
 849}
 850
 851static int __bio_iov_bvec_add_pages(struct bio *bio, struct iov_iter *iter)
 852{
 853        const struct bio_vec *bv = iter->bvec;
 854        unsigned int len;
 855        size_t size;
 856
 857        if (WARN_ON_ONCE(iter->iov_offset > bv->bv_len))
 858                return -EINVAL;
 859
 860        len = min_t(size_t, bv->bv_len - iter->iov_offset, iter->count);
 861        size = bio_add_page(bio, bv->bv_page, len,
 862                                bv->bv_offset + iter->iov_offset);
 863        if (unlikely(size != len))
 864                return -EINVAL;
 865        iov_iter_advance(iter, size);
 866        return 0;
 867}
 868
 869#define PAGE_PTRS_PER_BVEC     (sizeof(struct bio_vec) / sizeof(struct page *))
 870
 871/**
 872 * __bio_iov_iter_get_pages - pin user or kernel pages and add them to a bio
 873 * @bio: bio to add pages to
 874 * @iter: iov iterator describing the region to be mapped
 875 *
 876 * Pins pages from *iter and appends them to @bio's bvec array. The
 877 * pages will have to be released using put_page() when done.
 878 * For multi-segment *iter, this function only adds pages from the
 879 * the next non-empty segment of the iov iterator.
 880 */
 881static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
 882{
 883        unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt;
 884        unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt;
 885        struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt;
 886        struct page **pages = (struct page **)bv;
 887        bool same_page = false;
 888        ssize_t size, left;
 889        unsigned len, i;
 890        size_t offset;
 891
 892        /*
 893         * Move page array up in the allocated memory for the bio vecs as far as
 894         * possible so that we can start filling biovecs from the beginning
 895         * without overwriting the temporary page array.
 896        */
 897        BUILD_BUG_ON(PAGE_PTRS_PER_BVEC < 2);
 898        pages += entries_left * (PAGE_PTRS_PER_BVEC - 1);
 899
 900        size = iov_iter_get_pages(iter, pages, LONG_MAX, nr_pages, &offset);
 901        if (unlikely(size <= 0))
 902                return size ? size : -EFAULT;
 903
 904        for (left = size, i = 0; left > 0; left -= len, i++) {
 905                struct page *page = pages[i];
 906
 907                len = min_t(size_t, PAGE_SIZE - offset, left);
 908
 909                if (__bio_try_merge_page(bio, page, len, offset, &same_page)) {
 910                        if (same_page)
 911                                put_page(page);
 912                } else {
 913                        if (WARN_ON_ONCE(bio_full(bio, len)))
 914                                return -EINVAL;
 915                        __bio_add_page(bio, page, len, offset);
 916                }
 917                offset = 0;
 918        }
 919
 920        iov_iter_advance(iter, size);
 921        return 0;
 922}
 923
 924/**
 925 * bio_iov_iter_get_pages - add user or kernel pages to a bio
 926 * @bio: bio to add pages to
 927 * @iter: iov iterator describing the region to be added
 928 *
 929 * This takes either an iterator pointing to user memory, or one pointing to
 930 * kernel pages (BVEC iterator). If we're adding user pages, we pin them and
 931 * map them into the kernel. On IO completion, the caller should put those
 932 * pages. If we're adding kernel pages, and the caller told us it's safe to
 933 * do so, we just have to add the pages to the bio directly. We don't grab an
 934 * extra reference to those pages (the user should already have that), and we
 935 * don't put the page on IO completion. The caller needs to check if the bio is
 936 * flagged BIO_NO_PAGE_REF on IO completion. If it isn't, then pages should be
 937 * released.
 938 *
 939 * The function tries, but does not guarantee, to pin as many pages as
 940 * fit into the bio, or are requested in *iter, whatever is smaller. If
 941 * MM encounters an error pinning the requested pages, it stops. Error
 942 * is returned only if 0 pages could be pinned.
 943 */
 944int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter)
 945{
 946        const bool is_bvec = iov_iter_is_bvec(iter);
 947        int ret;
 948
 949        if (WARN_ON_ONCE(bio->bi_vcnt))
 950                return -EINVAL;
 951
 952        do {
 953                if (is_bvec)
 954                        ret = __bio_iov_bvec_add_pages(bio, iter);
 955                else
 956                        ret = __bio_iov_iter_get_pages(bio, iter);
 957        } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0));
 958
 959        if (is_bvec)
 960                bio_set_flag(bio, BIO_NO_PAGE_REF);
 961        return bio->bi_vcnt ? 0 : ret;
 962}
 963
 964static void submit_bio_wait_endio(struct bio *bio)
 965{
 966        complete(bio->bi_private);
 967}
 968
 969/**
 970 * submit_bio_wait - submit a bio, and wait until it completes
 971 * @bio: The &struct bio which describes the I/O
 972 *
 973 * Simple wrapper around submit_bio(). Returns 0 on success, or the error from
 974 * bio_endio() on failure.
 975 *
 976 * WARNING: Unlike to how submit_bio() is usually used, this function does not
 977 * result in bio reference to be consumed. The caller must drop the reference
 978 * on his own.
 979 */
 980int submit_bio_wait(struct bio *bio)
 981{
 982        DECLARE_COMPLETION_ONSTACK_MAP(done, bio->bi_disk->lockdep_map);
 983
 984        bio->bi_private = &done;
 985        bio->bi_end_io = submit_bio_wait_endio;
 986        bio->bi_opf |= REQ_SYNC;
 987        submit_bio(bio);
 988        wait_for_completion_io(&done);
 989
 990        return blk_status_to_errno(bio->bi_status);
 991}
 992EXPORT_SYMBOL(submit_bio_wait);
 993
 994/**
 995 * bio_advance - increment/complete a bio by some number of bytes
 996 * @bio:        bio to advance
 997 * @bytes:      number of bytes to complete
 998 *
 999 * This updates bi_sector, bi_size and bi_idx; if the number of bytes to
1000 * complete doesn't align with a bvec boundary, then bv_len and bv_offset will
1001 * be updated on the last bvec as well.
1002 *
1003 * @bio will then represent the remaining, uncompleted portion of the io.
1004 */
1005void bio_advance(struct bio *bio, unsigned bytes)
1006{
1007        if (bio_integrity(bio))
1008                bio_integrity_advance(bio, bytes);
1009
1010        bio_advance_iter(bio, &bio->bi_iter, bytes);
1011}
1012EXPORT_SYMBOL(bio_advance);
1013
1014void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
1015                        struct bio *src, struct bvec_iter *src_iter)
1016{
1017        struct bio_vec src_bv, dst_bv;
1018        void *src_p, *dst_p;
1019        unsigned bytes;
1020
1021        while (src_iter->bi_size && dst_iter->bi_size) {
1022                src_bv = bio_iter_iovec(src, *src_iter);
1023                dst_bv = bio_iter_iovec(dst, *dst_iter);
1024
1025                bytes = min(src_bv.bv_len, dst_bv.bv_len);
1026
1027                src_p = kmap_atomic(src_bv.bv_page);
1028                dst_p = kmap_atomic(dst_bv.bv_page);
1029
1030                memcpy(dst_p + dst_bv.bv_offset,
1031                       src_p + src_bv.bv_offset,
1032                       bytes);
1033
1034                kunmap_atomic(dst_p);
1035                kunmap_atomic(src_p);
1036
1037                flush_dcache_page(dst_bv.bv_page);
1038
1039                bio_advance_iter(src, src_iter, bytes);
1040                bio_advance_iter(dst, dst_iter, bytes);
1041        }
1042}
1043EXPORT_SYMBOL(bio_copy_data_iter);
1044
1045/**
1046 * bio_copy_data - copy contents of data buffers from one bio to another
1047 * @src: source bio
1048 * @dst: destination bio
1049 *
1050 * Stops when it reaches the end of either @src or @dst - that is, copies
1051 * min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of bios).
1052 */
1053void bio_copy_data(struct bio *dst, struct bio *src)
1054{
1055        struct bvec_iter src_iter = src->bi_iter;
1056        struct bvec_iter dst_iter = dst->bi_iter;
1057
1058        bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
1059}
1060EXPORT_SYMBOL(bio_copy_data);
1061
1062/**
1063 * bio_list_copy_data - copy contents of data buffers from one chain of bios to
1064 * another
1065 * @src: source bio list
1066 * @dst: destination bio list
1067 *
1068 * Stops when it reaches the end of either the @src list or @dst list - that is,
1069 * copies min(src->bi_size, dst->bi_size) bytes (or the equivalent for lists of
1070 * bios).
1071 */
1072void bio_list_copy_data(struct bio *dst, struct bio *src)
1073{
1074        struct bvec_iter src_iter = src->bi_iter;
1075        struct bvec_iter dst_iter = dst->bi_iter;
1076
1077        while (1) {
1078                if (!src_iter.bi_size) {
1079                        src = src->bi_next;
1080                        if (!src)
1081                                break;
1082
1083                        src_iter = src->bi_iter;
1084                }
1085
1086                if (!dst_iter.bi_size) {
1087                        dst = dst->bi_next;
1088                        if (!dst)
1089                                break;
1090
1091                        dst_iter = dst->bi_iter;
1092                }
1093
1094                bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
1095        }
1096}
1097EXPORT_SYMBOL(bio_list_copy_data);
1098
1099struct bio_map_data {
1100        int is_our_pages;
1101        struct iov_iter iter;
1102        struct iovec iov[];
1103};
1104
1105static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data,
1106                                               gfp_t gfp_mask)
1107{
1108        struct bio_map_data *bmd;
1109        if (data->nr_segs > UIO_MAXIOV)
1110                return NULL;
1111
1112        bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask);
1113        if (!bmd)
1114                return NULL;
1115        memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs);
1116        bmd->iter = *data;
1117        bmd->iter.iov = bmd->iov;
1118        return bmd;
1119}
1120
1121/**
1122 * bio_copy_from_iter - copy all pages from iov_iter to bio
1123 * @bio: The &struct bio which describes the I/O as destination
1124 * @iter: iov_iter as source
1125 *
1126 * Copy all pages from iov_iter to bio.
1127 * Returns 0 on success, or error on failure.
1128 */
1129static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter)
1130{
1131        struct bio_vec *bvec;
1132        struct bvec_iter_all iter_all;
1133
1134        bio_for_each_segment_all(bvec, bio, iter_all) {
1135                ssize_t ret;
1136
1137                ret = copy_page_from_iter(bvec->bv_page,
1138                                          bvec->bv_offset,
1139                                          bvec->bv_len,
1140                                          iter);
1141
1142                if (!iov_iter_count(iter))
1143                        break;
1144
1145                if (ret < bvec->bv_len)
1146                        return -EFAULT;
1147        }
1148
1149        return 0;
1150}
1151
1152/**
1153 * bio_copy_to_iter - copy all pages from bio to iov_iter
1154 * @bio: The &struct bio which describes the I/O as source
1155 * @iter: iov_iter as destination
1156 *
1157 * Copy all pages from bio to iov_iter.
1158 * Returns 0 on success, or error on failure.
1159 */
1160static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter)
1161{
1162        struct bio_vec *bvec;
1163        struct bvec_iter_all iter_all;
1164
1165        bio_for_each_segment_all(bvec, bio, iter_all) {
1166                ssize_t ret;
1167
1168                ret = copy_page_to_iter(bvec->bv_page,
1169                                        bvec->bv_offset,
1170                                        bvec->bv_len,
1171                                        &iter);
1172
1173                if (!iov_iter_count(&iter))
1174                        break;
1175
1176                if (ret < bvec->bv_len)
1177                        return -EFAULT;
1178        }
1179
1180        return 0;
1181}
1182
1183void bio_free_pages(struct bio *bio)
1184{
1185        struct bio_vec *bvec;
1186        struct bvec_iter_all iter_all;
1187
1188        bio_for_each_segment_all(bvec, bio, iter_all)
1189                __free_page(bvec->bv_page);
1190}
1191EXPORT_SYMBOL(bio_free_pages);
1192
1193/**
1194 *      bio_uncopy_user -       finish previously mapped bio
1195 *      @bio: bio being terminated
1196 *
1197 *      Free pages allocated from bio_copy_user_iov() and write back data
1198 *      to user space in case of a read.
1199 */
1200int bio_uncopy_user(struct bio *bio)
1201{
1202        struct bio_map_data *bmd = bio->bi_private;
1203        int ret = 0;
1204
1205        if (!bio_flagged(bio, BIO_NULL_MAPPED)) {
1206                /*
1207                 * if we're in a workqueue, the request is orphaned, so
1208                 * don't copy into a random user address space, just free
1209                 * and return -EINTR so user space doesn't expect any data.
1210                 */
1211                if (!current->mm)
1212                        ret = -EINTR;
1213                else if (bio_data_dir(bio) == READ)
1214                        ret = bio_copy_to_iter(bio, bmd->iter);
1215                if (bmd->is_our_pages)
1216                        bio_free_pages(bio);
1217        }
1218        kfree(bmd);
1219        bio_put(bio);
1220        return ret;
1221}
1222
1223/**
1224 *      bio_copy_user_iov       -       copy user data to bio
1225 *      @q:             destination block queue
1226 *      @map_data:      pointer to the rq_map_data holding pages (if necessary)
1227 *      @iter:          iovec iterator
1228 *      @gfp_mask:      memory allocation flags
1229 *
1230 *      Prepares and returns a bio for indirect user io, bouncing data
1231 *      to/from kernel pages as necessary. Must be paired with
1232 *      call bio_uncopy_user() on io completion.
1233 */
1234struct bio *bio_copy_user_iov(struct request_queue *q,
1235                              struct rq_map_data *map_data,
1236                              struct iov_iter *iter,
1237                              gfp_t gfp_mask)
1238{
1239        struct bio_map_data *bmd;
1240        struct page *page;
1241        struct bio *bio;
1242        int i = 0, ret;
1243        int nr_pages;
1244        unsigned int len = iter->count;
1245        unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0;
1246
1247        bmd = bio_alloc_map_data(iter, gfp_mask);
1248        if (!bmd)
1249                return ERR_PTR(-ENOMEM);
1250
1251        /*
1252         * We need to do a deep copy of the iov_iter including the iovecs.
1253         * The caller provided iov might point to an on-stack or otherwise
1254         * shortlived one.
1255         */
1256        bmd->is_our_pages = map_data ? 0 : 1;
1257
1258        nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE);
1259        if (nr_pages > BIO_MAX_PAGES)
1260                nr_pages = BIO_MAX_PAGES;
1261
1262        ret = -ENOMEM;
1263        bio = bio_kmalloc(gfp_mask, nr_pages);
1264        if (!bio)
1265                goto out_bmd;
1266
1267        ret = 0;
1268
1269        if (map_data) {
1270                nr_pages = 1 << map_data->page_order;
1271                i = map_data->offset / PAGE_SIZE;
1272        }
1273        while (len) {
1274                unsigned int bytes = PAGE_SIZE;
1275
1276                bytes -= offset;
1277
1278                if (bytes > len)
1279                        bytes = len;
1280
1281                if (map_data) {
1282                        if (i == map_data->nr_entries * nr_pages) {
1283                                ret = -ENOMEM;
1284                                break;
1285                        }
1286
1287                        page = map_data->pages[i / nr_pages];
1288                        page += (i % nr_pages);
1289
1290                        i++;
1291                } else {
1292                        page = alloc_page(q->bounce_gfp | gfp_mask);
1293                        if (!page) {
1294                                ret = -ENOMEM;
1295                                break;
1296                        }
1297                }
1298
1299                if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes) {
1300                        if (!map_data)
1301                                __free_page(page);
1302                        break;
1303                }
1304
1305                len -= bytes;
1306                offset = 0;
1307        }
1308
1309        if (ret)
1310                goto cleanup;
1311
1312        if (map_data)
1313                map_data->offset += bio->bi_iter.bi_size;
1314
1315        /*
1316         * success
1317         */
1318        if ((iov_iter_rw(iter) == WRITE && (!map_data || !map_data->null_mapped)) ||
1319            (map_data && map_data->from_user)) {
1320                ret = bio_copy_from_iter(bio, iter);
1321                if (ret)
1322                        goto cleanup;
1323        } else {
1324                if (bmd->is_our_pages)
1325                        zero_fill_bio(bio);
1326                iov_iter_advance(iter, bio->bi_iter.bi_size);
1327        }
1328
1329        bio->bi_private = bmd;
1330        if (map_data && map_data->null_mapped)
1331                bio_set_flag(bio, BIO_NULL_MAPPED);
1332        return bio;
1333cleanup:
1334        if (!map_data)
1335                bio_free_pages(bio);
1336        bio_put(bio);
1337out_bmd:
1338        kfree(bmd);
1339        return ERR_PTR(ret);
1340}
1341
1342/**
1343 *      bio_map_user_iov - map user iovec into bio
1344 *      @q:             the struct request_queue for the bio
1345 *      @iter:          iovec iterator
1346 *      @gfp_mask:      memory allocation flags
1347 *
1348 *      Map the user space address into a bio suitable for io to a block
1349 *      device. Returns an error pointer in case of error.
1350 */
1351struct bio *bio_map_user_iov(struct request_queue *q,
1352                             struct iov_iter *iter,
1353                             gfp_t gfp_mask)
1354{
1355        int j;
1356        struct bio *bio;
1357        int ret;
1358
1359        if (!iov_iter_count(iter))
1360                return ERR_PTR(-EINVAL);
1361
1362        bio = bio_kmalloc(gfp_mask, iov_iter_npages(iter, BIO_MAX_PAGES));
1363        if (!bio)
1364                return ERR_PTR(-ENOMEM);
1365
1366        while (iov_iter_count(iter)) {
1367                struct page **pages;
1368                ssize_t bytes;
1369                size_t offs, added = 0;
1370                int npages;
1371
1372                bytes = iov_iter_get_pages_alloc(iter, &pages, LONG_MAX, &offs);
1373                if (unlikely(bytes <= 0)) {
1374                        ret = bytes ? bytes : -EFAULT;
1375                        goto out_unmap;
1376                }
1377
1378                npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE);
1379
1380                if (unlikely(offs & queue_dma_alignment(q))) {
1381                        ret = -EINVAL;
1382                        j = 0;
1383                } else {
1384                        for (j = 0; j < npages; j++) {
1385                                struct page *page = pages[j];
1386                                unsigned int n = PAGE_SIZE - offs;
1387
1388                                if (n > bytes)
1389                                        n = bytes;
1390
1391                                if (!__bio_add_pc_page(q, bio, page, n, offs,
1392                                                        true))
1393                                        break;
1394
1395                                added += n;
1396                                bytes -= n;
1397                                offs = 0;
1398                        }
1399                        iov_iter_advance(iter, added);
1400                }
1401                /*
1402                 * release the pages we didn't map into the bio, if any
1403                 */
1404                while (j < npages)
1405                        put_page(pages[j++]);
1406                kvfree(pages);
1407                /* couldn't stuff something into bio? */
1408                if (bytes)
1409                        break;
1410        }
1411
1412        bio_set_flag(bio, BIO_USER_MAPPED);
1413
1414        /*
1415         * subtle -- if bio_map_user_iov() ended up bouncing a bio,
1416         * it would normally disappear when its bi_end_io is run.
1417         * however, we need it for the unmap, so grab an extra
1418         * reference to it
1419         */
1420        bio_get(bio);
1421        return bio;
1422
1423 out_unmap:
1424        bio_release_pages(bio, false);
1425        bio_put(bio);
1426        return ERR_PTR(ret);
1427}
1428
1429/**
1430 *      bio_unmap_user  -       unmap a bio
1431 *      @bio:           the bio being unmapped
1432 *
1433 *      Unmap a bio previously mapped by bio_map_user_iov(). Must be called from
1434 *      process context.
1435 *
1436 *      bio_unmap_user() may sleep.
1437 */
1438void bio_unmap_user(struct bio *bio)
1439{
1440        bio_release_pages(bio, bio_data_dir(bio) == READ);
1441        bio_put(bio);
1442        bio_put(bio);
1443}
1444
1445static void bio_invalidate_vmalloc_pages(struct bio *bio)
1446{
1447#ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE
1448        if (bio->bi_private && !op_is_write(bio_op(bio))) {
1449                unsigned long i, len = 0;
1450
1451                for (i = 0; i < bio->bi_vcnt; i++)
1452                        len += bio->bi_io_vec[i].bv_len;
1453                invalidate_kernel_vmap_range(bio->bi_private, len);
1454        }
1455#endif
1456}
1457
1458static void bio_map_kern_endio(struct bio *bio)
1459{
1460        bio_invalidate_vmalloc_pages(bio);
1461        bio_put(bio);
1462}
1463
1464/**
1465 *      bio_map_kern    -       map kernel address into bio
1466 *      @q: the struct request_queue for the bio
1467 *      @data: pointer to buffer to map
1468 *      @len: length in bytes
1469 *      @gfp_mask: allocation flags for bio allocation
1470 *
1471 *      Map the kernel address into a bio suitable for io to a block
1472 *      device. Returns an error pointer in case of error.
1473 */
1474struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
1475                         gfp_t gfp_mask)
1476{
1477        unsigned long kaddr = (unsigned long)data;
1478        unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1479        unsigned long start = kaddr >> PAGE_SHIFT;
1480        const int nr_pages = end - start;
1481        bool is_vmalloc = is_vmalloc_addr(data);
1482        struct page *page;
1483        int offset, i;
1484        struct bio *bio;
1485
1486        bio = bio_kmalloc(gfp_mask, nr_pages);
1487        if (!bio)
1488                return ERR_PTR(-ENOMEM);
1489
1490        if (is_vmalloc) {
1491                flush_kernel_vmap_range(data, len);
1492                bio->bi_private = data;
1493        }
1494
1495        offset = offset_in_page(kaddr);
1496        for (i = 0; i < nr_pages; i++) {
1497                unsigned int bytes = PAGE_SIZE - offset;
1498
1499                if (len <= 0)
1500                        break;
1501
1502                if (bytes > len)
1503                        bytes = len;
1504
1505                if (!is_vmalloc)
1506                        page = virt_to_page(data);
1507                else
1508                        page = vmalloc_to_page(data);
1509                if (bio_add_pc_page(q, bio, page, bytes,
1510                                    offset) < bytes) {
1511                        /* we don't support partial mappings */
1512                        bio_put(bio);
1513                        return ERR_PTR(-EINVAL);
1514                }
1515
1516                data += bytes;
1517                len -= bytes;
1518                offset = 0;
1519        }
1520
1521        bio->bi_end_io = bio_map_kern_endio;
1522        return bio;
1523}
1524EXPORT_SYMBOL(bio_map_kern);
1525
1526static void bio_copy_kern_endio(struct bio *bio)
1527{
1528        bio_free_pages(bio);
1529        bio_put(bio);
1530}
1531
1532static void bio_copy_kern_endio_read(struct bio *bio)
1533{
1534        char *p = bio->bi_private;
1535        struct bio_vec *bvec;
1536        struct bvec_iter_all iter_all;
1537
1538        bio_for_each_segment_all(bvec, bio, iter_all) {
1539                memcpy(p, page_address(bvec->bv_page), bvec->bv_len);
1540                p += bvec->bv_len;
1541        }
1542
1543        bio_copy_kern_endio(bio);
1544}
1545
1546/**
1547 *      bio_copy_kern   -       copy kernel address into bio
1548 *      @q: the struct request_queue for the bio
1549 *      @data: pointer to buffer to copy
1550 *      @len: length in bytes
1551 *      @gfp_mask: allocation flags for bio and page allocation
1552 *      @reading: data direction is READ
1553 *
1554 *      copy the kernel address into a bio suitable for io to a block
1555 *      device. Returns an error pointer in case of error.
1556 */
1557struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len,
1558                          gfp_t gfp_mask, int reading)
1559{
1560        unsigned long kaddr = (unsigned long)data;
1561        unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1562        unsigned long start = kaddr >> PAGE_SHIFT;
1563        struct bio *bio;
1564        void *p = data;
1565        int nr_pages = 0;
1566
1567        /*
1568         * Overflow, abort
1569         */
1570        if (end < start)
1571                return ERR_PTR(-EINVAL);
1572
1573        nr_pages = end - start;
1574        bio = bio_kmalloc(gfp_mask, nr_pages);
1575        if (!bio)
1576                return ERR_PTR(-ENOMEM);
1577
1578        while (len) {
1579                struct page *page;
1580                unsigned int bytes = PAGE_SIZE;
1581
1582                if (bytes > len)
1583                        bytes = len;
1584
1585                page = alloc_page(q->bounce_gfp | gfp_mask);
1586                if (!page)
1587                        goto cleanup;
1588
1589                if (!reading)
1590                        memcpy(page_address(page), p, bytes);
1591
1592                if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes)
1593                        break;
1594
1595                len -= bytes;
1596                p += bytes;
1597        }
1598
1599        if (reading) {
1600                bio->bi_end_io = bio_copy_kern_endio_read;
1601                bio->bi_private = data;
1602        } else {
1603                bio->bi_end_io = bio_copy_kern_endio;
1604        }
1605
1606        return bio;
1607
1608cleanup:
1609        bio_free_pages(bio);
1610        bio_put(bio);
1611        return ERR_PTR(-ENOMEM);
1612}
1613
1614/*
1615 * bio_set_pages_dirty() and bio_check_pages_dirty() are support functions
1616 * for performing direct-IO in BIOs.
1617 *
1618 * The problem is that we cannot run set_page_dirty() from interrupt context
1619 * because the required locks are not interrupt-safe.  So what we can do is to
1620 * mark the pages dirty _before_ performing IO.  And in interrupt context,
1621 * check that the pages are still dirty.   If so, fine.  If not, redirty them
1622 * in process context.
1623 *
1624 * We special-case compound pages here: normally this means reads into hugetlb
1625 * pages.  The logic in here doesn't really work right for compound pages
1626 * because the VM does not uniformly chase down the head page in all cases.
1627 * But dirtiness of compound pages is pretty meaningless anyway: the VM doesn't
1628 * handle them at all.  So we skip compound pages here at an early stage.
1629 *
1630 * Note that this code is very hard to test under normal circumstances because
1631 * direct-io pins the pages with get_user_pages().  This makes
1632 * is_page_cache_freeable return false, and the VM will not clean the pages.
1633 * But other code (eg, flusher threads) could clean the pages if they are mapped
1634 * pagecache.
1635 *
1636 * Simply disabling the call to bio_set_pages_dirty() is a good way to test the
1637 * deferred bio dirtying paths.
1638 */
1639
1640/*
1641 * bio_set_pages_dirty() will mark all the bio's pages as dirty.
1642 */
1643void bio_set_pages_dirty(struct bio *bio)
1644{
1645        struct bio_vec *bvec;
1646        struct bvec_iter_all iter_all;
1647
1648        bio_for_each_segment_all(bvec, bio, iter_all) {
1649                if (!PageCompound(bvec->bv_page))
1650                        set_page_dirty_lock(bvec->bv_page);
1651        }
1652}
1653
1654/*
1655 * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
1656 * If they are, then fine.  If, however, some pages are clean then they must
1657 * have been written out during the direct-IO read.  So we take another ref on
1658 * the BIO and re-dirty the pages in process context.
1659 *
1660 * It is expected that bio_check_pages_dirty() will wholly own the BIO from
1661 * here on.  It will run one put_page() against each page and will run one
1662 * bio_put() against the BIO.
1663 */
1664
1665static void bio_dirty_fn(struct work_struct *work);
1666
1667static DECLARE_WORK(bio_dirty_work, bio_dirty_fn);
1668static DEFINE_SPINLOCK(bio_dirty_lock);
1669static struct bio *bio_dirty_list;
1670
1671/*
1672 * This runs in process context
1673 */
1674static void bio_dirty_fn(struct work_struct *work)
1675{
1676        struct bio *bio, *next;
1677
1678        spin_lock_irq(&bio_dirty_lock);
1679        next = bio_dirty_list;
1680        bio_dirty_list = NULL;
1681        spin_unlock_irq(&bio_dirty_lock);
1682
1683        while ((bio = next) != NULL) {
1684                next = bio->bi_private;
1685
1686                bio_release_pages(bio, true);
1687                bio_put(bio);
1688        }
1689}
1690
1691void bio_check_pages_dirty(struct bio *bio)
1692{
1693        struct bio_vec *bvec;
1694        unsigned long flags;
1695        struct bvec_iter_all iter_all;
1696
1697        bio_for_each_segment_all(bvec, bio, iter_all) {
1698                if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page))
1699                        goto defer;
1700        }
1701
1702        bio_release_pages(bio, false);
1703        bio_put(bio);
1704        return;
1705defer:
1706        spin_lock_irqsave(&bio_dirty_lock, flags);
1707        bio->bi_private = bio_dirty_list;
1708        bio_dirty_list = bio;
1709        spin_unlock_irqrestore(&bio_dirty_lock, flags);
1710        schedule_work(&bio_dirty_work);
1711}
1712
1713void update_io_ticks(struct hd_struct *part, unsigned long now)
1714{
1715        unsigned long stamp;
1716again:
1717        stamp = READ_ONCE(part->stamp);
1718        if (unlikely(stamp != now)) {
1719                if (likely(cmpxchg(&part->stamp, stamp, now) == stamp)) {
1720                        __part_stat_add(part, io_ticks, 1);
1721                }
1722        }
1723        if (part->partno) {
1724                part = &part_to_disk(part)->part0;
1725                goto again;
1726        }
1727}
1728
1729void generic_start_io_acct(struct request_queue *q, int op,
1730                           unsigned long sectors, struct hd_struct *part)
1731{
1732        const int sgrp = op_stat_group(op);
1733
1734        part_stat_lock();
1735
1736        update_io_ticks(part, jiffies);
1737        part_stat_inc(part, ios[sgrp]);
1738        part_stat_add(part, sectors[sgrp], sectors);
1739        part_inc_in_flight(q, part, op_is_write(op));
1740
1741        part_stat_unlock();
1742}
1743EXPORT_SYMBOL(generic_start_io_acct);
1744
1745void generic_end_io_acct(struct request_queue *q, int req_op,
1746                         struct hd_struct *part, unsigned long start_time)
1747{
1748        unsigned long now = jiffies;
1749        unsigned long duration = now - start_time;
1750        const int sgrp = op_stat_group(req_op);
1751
1752        part_stat_lock();
1753
1754        update_io_ticks(part, now);
1755        part_stat_add(part, nsecs[sgrp], jiffies_to_nsecs(duration));
1756        part_stat_add(part, time_in_queue, duration);
1757        part_dec_in_flight(q, part, op_is_write(req_op));
1758
1759        part_stat_unlock();
1760}
1761EXPORT_SYMBOL(generic_end_io_acct);
1762
1763static inline bool bio_remaining_done(struct bio *bio)
1764{
1765        /*
1766         * If we're not chaining, then ->__bi_remaining is always 1 and
1767         * we always end io on the first invocation.
1768         */
1769        if (!bio_flagged(bio, BIO_CHAIN))
1770                return true;
1771
1772        BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
1773
1774        if (atomic_dec_and_test(&bio->__bi_remaining)) {
1775                bio_clear_flag(bio, BIO_CHAIN);
1776                return true;
1777        }
1778
1779        return false;
1780}
1781
1782/**
1783 * bio_endio - end I/O on a bio
1784 * @bio:        bio
1785 *
1786 * Description:
1787 *   bio_endio() will end I/O on the whole bio. bio_endio() is the preferred
1788 *   way to end I/O on a bio. No one should call bi_end_io() directly on a
1789 *   bio unless they own it and thus know that it has an end_io function.
1790 *
1791 *   bio_endio() can be called several times on a bio that has been chained
1792 *   using bio_chain().  The ->bi_end_io() function will only be called the
1793 *   last time.  At this point the BLK_TA_COMPLETE tracing event will be
1794 *   generated if BIO_TRACE_COMPLETION is set.
1795 **/
1796void bio_endio(struct bio *bio)
1797{
1798again:
1799        if (!bio_remaining_done(bio))
1800                return;
1801        if (!bio_integrity_endio(bio))
1802                return;
1803
1804        if (bio->bi_disk)
1805                rq_qos_done_bio(bio->bi_disk->queue, bio);
1806
1807        /*
1808         * Need to have a real endio function for chained bios, otherwise
1809         * various corner cases will break (like stacking block devices that
1810         * save/restore bi_end_io) - however, we want to avoid unbounded
1811         * recursion and blowing the stack. Tail call optimization would
1812         * handle this, but compiling with frame pointers also disables
1813         * gcc's sibling call optimization.
1814         */
1815        if (bio->bi_end_io == bio_chain_endio) {
1816                bio = __bio_chain_endio(bio);
1817                goto again;
1818        }
1819
1820        if (bio->bi_disk && bio_flagged(bio, BIO_TRACE_COMPLETION)) {
1821                trace_block_bio_complete(bio->bi_disk->queue, bio,
1822                                         blk_status_to_errno(bio->bi_status));
1823                bio_clear_flag(bio, BIO_TRACE_COMPLETION);
1824        }
1825
1826        blk_throtl_bio_endio(bio);
1827        /* release cgroup info */
1828        bio_uninit(bio);
1829        if (bio->bi_end_io)
1830                bio->bi_end_io(bio);
1831}
1832EXPORT_SYMBOL(bio_endio);
1833
1834/**
1835 * bio_split - split a bio
1836 * @bio:        bio to split
1837 * @sectors:    number of sectors to split from the front of @bio
1838 * @gfp:        gfp mask
1839 * @bs:         bio set to allocate from
1840 *
1841 * Allocates and returns a new bio which represents @sectors from the start of
1842 * @bio, and updates @bio to represent the remaining sectors.
1843 *
1844 * Unless this is a discard request the newly allocated bio will point
1845 * to @bio's bi_io_vec; it is the caller's responsibility to ensure that
1846 * @bio is not freed before the split.
1847 */
1848struct bio *bio_split(struct bio *bio, int sectors,
1849                      gfp_t gfp, struct bio_set *bs)
1850{
1851        struct bio *split;
1852
1853        BUG_ON(sectors <= 0);
1854        BUG_ON(sectors >= bio_sectors(bio));
1855
1856        split = bio_clone_fast(bio, gfp, bs);
1857        if (!split)
1858                return NULL;
1859
1860        split->bi_iter.bi_size = sectors << 9;
1861
1862        if (bio_integrity(split))
1863                bio_integrity_trim(split);
1864
1865        bio_advance(bio, split->bi_iter.bi_size);
1866
1867        if (bio_flagged(bio, BIO_TRACE_COMPLETION))
1868                bio_set_flag(split, BIO_TRACE_COMPLETION);
1869
1870        return split;
1871}
1872EXPORT_SYMBOL(bio_split);
1873
1874/**
1875 * bio_trim - trim a bio
1876 * @bio:        bio to trim
1877 * @offset:     number of sectors to trim from the front of @bio
1878 * @size:       size we want to trim @bio to, in sectors
1879 */
1880void bio_trim(struct bio *bio, int offset, int size)
1881{
1882        /* 'bio' is a cloned bio which we need to trim to match
1883         * the given offset and size.
1884         */
1885
1886        size <<= 9;
1887        if (offset == 0 && size == bio->bi_iter.bi_size)
1888                return;
1889
1890        bio_advance(bio, offset << 9);
1891        bio->bi_iter.bi_size = size;
1892
1893        if (bio_integrity(bio))
1894                bio_integrity_trim(bio);
1895
1896}
1897EXPORT_SYMBOL_GPL(bio_trim);
1898
1899/*
1900 * create memory pools for biovec's in a bio_set.
1901 * use the global biovec slabs created for general use.
1902 */
1903int biovec_init_pool(mempool_t *pool, int pool_entries)
1904{
1905        struct biovec_slab *bp = bvec_slabs + BVEC_POOL_MAX;
1906
1907        return mempool_init_slab_pool(pool, pool_entries, bp->slab);
1908}
1909
1910/*
1911 * bioset_exit - exit a bioset initialized with bioset_init()
1912 *
1913 * May be called on a zeroed but uninitialized bioset (i.e. allocated with
1914 * kzalloc()).
1915 */
1916void bioset_exit(struct bio_set *bs)
1917{
1918        if (bs->rescue_workqueue)
1919                destroy_workqueue(bs->rescue_workqueue);
1920        bs->rescue_workqueue = NULL;
1921
1922        mempool_exit(&bs->bio_pool);
1923        mempool_exit(&bs->bvec_pool);
1924
1925        bioset_integrity_free(bs);
1926        if (bs->bio_slab)
1927                bio_put_slab(bs);
1928        bs->bio_slab = NULL;
1929}
1930EXPORT_SYMBOL(bioset_exit);
1931
1932/**
1933 * bioset_init - Initialize a bio_set
1934 * @bs:         pool to initialize
1935 * @pool_size:  Number of bio and bio_vecs to cache in the mempool
1936 * @front_pad:  Number of bytes to allocate in front of the returned bio
1937 * @flags:      Flags to modify behavior, currently %BIOSET_NEED_BVECS
1938 *              and %BIOSET_NEED_RESCUER
1939 *
1940 * Description:
1941 *    Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller
1942 *    to ask for a number of bytes to be allocated in front of the bio.
1943 *    Front pad allocation is useful for embedding the bio inside
1944 *    another structure, to avoid allocating extra data to go with the bio.
1945 *    Note that the bio must be embedded at the END of that structure always,
1946 *    or things will break badly.
1947 *    If %BIOSET_NEED_BVECS is set in @flags, a separate pool will be allocated
1948 *    for allocating iovecs.  This pool is not needed e.g. for bio_clone_fast().
1949 *    If %BIOSET_NEED_RESCUER is set, a workqueue is created which can be used to
1950 *    dispatch queued requests when the mempool runs out of space.
1951 *
1952 */
1953int bioset_init(struct bio_set *bs,
1954                unsigned int pool_size,
1955                unsigned int front_pad,
1956                int flags)
1957{
1958        unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec);
1959
1960        bs->front_pad = front_pad;
1961
1962        spin_lock_init(&bs->rescue_lock);
1963        bio_list_init(&bs->rescue_list);
1964        INIT_WORK(&bs->rescue_work, bio_alloc_rescue);
1965
1966        bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);
1967        if (!bs->bio_slab)
1968                return -ENOMEM;
1969
1970        if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab))
1971                goto bad;
1972
1973        if ((flags & BIOSET_NEED_BVECS) &&
1974            biovec_init_pool(&bs->bvec_pool, pool_size))
1975                goto bad;
1976
1977        if (!(flags & BIOSET_NEED_RESCUER))
1978                return 0;
1979
1980        bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);
1981        if (!bs->rescue_workqueue)
1982                goto bad;
1983
1984        return 0;
1985bad:
1986        bioset_exit(bs);
1987        return -ENOMEM;
1988}
1989EXPORT_SYMBOL(bioset_init);
1990
1991/*
1992 * Initialize and setup a new bio_set, based on the settings from
1993 * another bio_set.
1994 */
1995int bioset_init_from_src(struct bio_set *bs, struct bio_set *src)
1996{
1997        int flags;
1998
1999        flags = 0;
2000        if (src->bvec_pool.min_nr)
2001                flags |= BIOSET_NEED_BVECS;
2002        if (src->rescue_workqueue)
2003                flags |= BIOSET_NEED_RESCUER;
2004
2005        return bioset_init(bs, src->bio_pool.min_nr, src->front_pad, flags);
2006}
2007EXPORT_SYMBOL(bioset_init_from_src);
2008
2009#ifdef CONFIG_BLK_CGROUP
2010
2011/**
2012 * bio_disassociate_blkg - puts back the blkg reference if associated
2013 * @bio: target bio
2014 *
2015 * Helper to disassociate the blkg from @bio if a blkg is associated.
2016 */
2017void bio_disassociate_blkg(struct bio *bio)
2018{
2019        if (bio->bi_blkg) {
2020                blkg_put(bio->bi_blkg);
2021                bio->bi_blkg = NULL;
2022        }
2023}
2024EXPORT_SYMBOL_GPL(bio_disassociate_blkg);
2025
2026/**
2027 * __bio_associate_blkg - associate a bio with the a blkg
2028 * @bio: target bio
2029 * @blkg: the blkg to associate
2030 *
2031 * This tries to associate @bio with the specified @blkg.  Association failure
2032 * is handled by walking up the blkg tree.  Therefore, the blkg associated can
2033 * be anything between @blkg and the root_blkg.  This situation only happens
2034 * when a cgroup is dying and then the remaining bios will spill to the closest
2035 * alive blkg.
2036 *
2037 * A reference will be taken on the @blkg and will be released when @bio is
2038 * freed.
2039 */
2040static void __bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg)
2041{
2042        bio_disassociate_blkg(bio);
2043
2044        bio->bi_blkg = blkg_tryget_closest(blkg);
2045}
2046
2047/**
2048 * bio_associate_blkg_from_css - associate a bio with a specified css
2049 * @bio: target bio
2050 * @css: target css
2051 *
2052 * Associate @bio with the blkg found by combining the css's blkg and the
2053 * request_queue of the @bio.  This falls back to the queue's root_blkg if
2054 * the association fails with the css.
2055 */
2056void bio_associate_blkg_from_css(struct bio *bio,
2057                                 struct cgroup_subsys_state *css)
2058{
2059        struct request_queue *q = bio->bi_disk->queue;
2060        struct blkcg_gq *blkg;
2061
2062        rcu_read_lock();
2063
2064        if (!css || !css->parent)
2065                blkg = q->root_blkg;
2066        else
2067                blkg = blkg_lookup_create(css_to_blkcg(css), q);
2068
2069        __bio_associate_blkg(bio, blkg);
2070
2071        rcu_read_unlock();
2072}
2073EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);
2074
2075#ifdef CONFIG_MEMCG
2076/**
2077 * bio_associate_blkg_from_page - associate a bio with the page's blkg
2078 * @bio: target bio
2079 * @page: the page to lookup the blkcg from
2080 *
2081 * Associate @bio with the blkg from @page's owning memcg and the respective
2082 * request_queue.  If cgroup_e_css returns %NULL, fall back to the queue's
2083 * root_blkg.
2084 */
2085void bio_associate_blkg_from_page(struct bio *bio, struct page *page)
2086{
2087        struct cgroup_subsys_state *css;
2088
2089        if (!page->mem_cgroup)
2090                return;
2091
2092        rcu_read_lock();
2093
2094        css = cgroup_e_css(page->mem_cgroup->css.cgroup, &io_cgrp_subsys);
2095        bio_associate_blkg_from_css(bio, css);
2096
2097        rcu_read_unlock();
2098}
2099#endif /* CONFIG_MEMCG */
2100
2101/**
2102 * bio_associate_blkg - associate a bio with a blkg
2103 * @bio: target bio
2104 *
2105 * Associate @bio with the blkg found from the bio's css and request_queue.
2106 * If one is not found, bio_lookup_blkg() creates the blkg.  If a blkg is
2107 * already associated, the css is reused and association redone as the
2108 * request_queue may have changed.
2109 */
2110void bio_associate_blkg(struct bio *bio)
2111{
2112        struct cgroup_subsys_state *css;
2113
2114        rcu_read_lock();
2115
2116        if (bio->bi_blkg)
2117                css = &bio_blkcg(bio)->css;
2118        else
2119                css = blkcg_css();
2120
2121        bio_associate_blkg_from_css(bio, css);
2122
2123        rcu_read_unlock();
2124}
2125EXPORT_SYMBOL_GPL(bio_associate_blkg);
2126
2127/**
2128 * bio_clone_blkg_association - clone blkg association from src to dst bio
2129 * @dst: destination bio
2130 * @src: source bio
2131 */
2132void bio_clone_blkg_association(struct bio *dst, struct bio *src)
2133{
2134        rcu_read_lock();
2135
2136        if (src->bi_blkg)
2137                __bio_associate_blkg(dst, src->bi_blkg);
2138
2139        rcu_read_unlock();
2140}
2141EXPORT_SYMBOL_GPL(bio_clone_blkg_association);
2142#endif /* CONFIG_BLK_CGROUP */
2143
2144static void __init biovec_init_slabs(void)
2145{
2146        int i;
2147
2148        for (i = 0; i < BVEC_POOL_NR; i++) {
2149                int size;
2150                struct biovec_slab *bvs = bvec_slabs + i;
2151
2152                if (bvs->nr_vecs <= BIO_INLINE_VECS) {
2153                        bvs->slab = NULL;
2154                        continue;
2155                }
2156
2157                size = bvs->nr_vecs * sizeof(struct bio_vec);
2158                bvs->slab = kmem_cache_create(bvs->name, size, 0,
2159                                SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2160        }
2161}
2162
2163static int __init init_bio(void)
2164{
2165        bio_slab_max = 2;
2166        bio_slab_nr = 0;
2167        bio_slabs = kcalloc(bio_slab_max, sizeof(struct bio_slab),
2168                            GFP_KERNEL);
2169
2170        BUILD_BUG_ON(BIO_FLAG_LAST > BVEC_POOL_OFFSET);
2171
2172        if (!bio_slabs)
2173                panic("bio: can't allocate bios\n");
2174
2175        bio_integrity_init();
2176        biovec_init_slabs();
2177
2178        if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS))
2179                panic("bio: can't allocate bios\n");
2180
2181        if (bioset_integrity_create(&fs_bio_set, BIO_POOL_SIZE))
2182                panic("bio: can't create integrity pool\n");
2183
2184        return 0;
2185}
2186subsys_initcall(init_bio);
2187