1
2
3
4
5#include <linux/kernel.h>
6#include <linux/module.h>
7#include <linux/bio.h>
8#include <linux/blkdev.h>
9#include <linux/scatterlist.h>
10
11#include <trace/events/block.h>
12
13#include "blk.h"
14
15static inline bool bio_will_gap(struct request_queue *q,
16 struct request *prev_rq, struct bio *prev, struct bio *next)
17{
18 struct bio_vec pb, nb;
19
20 if (!bio_has_data(prev) || !queue_virt_boundary(q))
21 return false;
22
23
24
25
26
27
28 if (prev_rq)
29 bio_get_first_bvec(prev_rq->bio, &pb);
30 else
31 bio_get_first_bvec(prev, &pb);
32 if (pb.bv_offset & queue_virt_boundary(q))
33 return true;
34
35
36
37
38
39
40
41
42
43
44 bio_get_last_bvec(prev, &pb);
45 bio_get_first_bvec(next, &nb);
46 if (biovec_phys_mergeable(q, &pb, &nb))
47 return false;
48 return __bvec_gap_to_prev(q, &pb, nb.bv_offset);
49}
50
51static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
52{
53 return bio_will_gap(req->q, req, req->biotail, bio);
54}
55
56static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
57{
58 return bio_will_gap(req->q, NULL, bio, req->bio);
59}
60
61static struct bio *blk_bio_discard_split(struct request_queue *q,
62 struct bio *bio,
63 struct bio_set *bs,
64 unsigned *nsegs)
65{
66 unsigned int max_discard_sectors, granularity;
67 int alignment;
68 sector_t tmp;
69 unsigned split_sectors;
70
71 *nsegs = 1;
72
73
74 granularity = max(q->limits.discard_granularity >> 9, 1U);
75
76 max_discard_sectors = min(q->limits.max_discard_sectors,
77 bio_allowed_max_sectors(q));
78 max_discard_sectors -= max_discard_sectors % granularity;
79
80 if (unlikely(!max_discard_sectors)) {
81
82 return NULL;
83 }
84
85 if (bio_sectors(bio) <= max_discard_sectors)
86 return NULL;
87
88 split_sectors = max_discard_sectors;
89
90
91
92
93
94 alignment = (q->limits.discard_alignment >> 9) % granularity;
95
96 tmp = bio->bi_iter.bi_sector + split_sectors - alignment;
97 tmp = sector_div(tmp, granularity);
98
99 if (split_sectors > tmp)
100 split_sectors -= tmp;
101
102 return bio_split(bio, split_sectors, GFP_NOIO, bs);
103}
104
105static struct bio *blk_bio_write_zeroes_split(struct request_queue *q,
106 struct bio *bio, struct bio_set *bs, unsigned *nsegs)
107{
108 *nsegs = 0;
109
110 if (!q->limits.max_write_zeroes_sectors)
111 return NULL;
112
113 if (bio_sectors(bio) <= q->limits.max_write_zeroes_sectors)
114 return NULL;
115
116 return bio_split(bio, q->limits.max_write_zeroes_sectors, GFP_NOIO, bs);
117}
118
119static struct bio *blk_bio_write_same_split(struct request_queue *q,
120 struct bio *bio,
121 struct bio_set *bs,
122 unsigned *nsegs)
123{
124 *nsegs = 1;
125
126 if (!q->limits.max_write_same_sectors)
127 return NULL;
128
129 if (bio_sectors(bio) <= q->limits.max_write_same_sectors)
130 return NULL;
131
132 return bio_split(bio, q->limits.max_write_same_sectors, GFP_NOIO, bs);
133}
134
135static inline unsigned get_max_io_size(struct request_queue *q,
136 struct bio *bio)
137{
138 unsigned sectors = blk_max_size_offset(q, bio->bi_iter.bi_sector);
139 unsigned mask = queue_logical_block_size(q) - 1;
140
141
142 sectors &= ~(mask >> 9);
143
144 return sectors;
145}
146
147static unsigned get_max_segment_size(struct request_queue *q,
148 unsigned offset)
149{
150 unsigned long mask = queue_segment_boundary(q);
151
152
153 if (mask == BLK_SEG_BOUNDARY_MASK)
154 return queue_max_segment_size(q);
155
156 return min_t(unsigned long, mask - (mask & offset) + 1,
157 queue_max_segment_size(q));
158}
159
160
161
162
163
164static bool bvec_split_segs(struct request_queue *q, struct bio_vec *bv,
165 unsigned *nsegs, unsigned *sectors, unsigned max_segs)
166{
167 unsigned len = bv->bv_len;
168 unsigned total_len = 0;
169 unsigned new_nsegs = 0, seg_size = 0;
170
171
172
173
174
175 while (len && new_nsegs + *nsegs < max_segs) {
176 seg_size = get_max_segment_size(q, bv->bv_offset + total_len);
177 seg_size = min(seg_size, len);
178
179 new_nsegs++;
180 total_len += seg_size;
181 len -= seg_size;
182
183 if ((bv->bv_offset + total_len) & queue_virt_boundary(q))
184 break;
185 }
186
187 if (new_nsegs) {
188 *nsegs += new_nsegs;
189 if (sectors)
190 *sectors += total_len >> 9;
191 }
192
193
194 return !!len;
195}
196
197static struct bio *blk_bio_segment_split(struct request_queue *q,
198 struct bio *bio,
199 struct bio_set *bs,
200 unsigned *segs)
201{
202 struct bio_vec bv, bvprv, *bvprvp = NULL;
203 struct bvec_iter iter;
204 unsigned nsegs = 0, sectors = 0;
205 const unsigned max_sectors = get_max_io_size(q, bio);
206 const unsigned max_segs = queue_max_segments(q);
207
208 bio_for_each_bvec(bv, bio, iter) {
209
210
211
212
213 if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset))
214 goto split;
215
216 if (sectors + (bv.bv_len >> 9) > max_sectors) {
217
218
219
220
221 if (nsegs < max_segs &&
222 sectors < max_sectors) {
223
224 bv.bv_len = (max_sectors - sectors) << 9;
225 bvec_split_segs(q, &bv, &nsegs,
226 §ors, max_segs);
227 }
228 goto split;
229 }
230
231 if (nsegs == max_segs)
232 goto split;
233
234 bvprv = bv;
235 bvprvp = &bvprv;
236
237 if (bv.bv_offset + bv.bv_len <= PAGE_SIZE) {
238 nsegs++;
239 sectors += bv.bv_len >> 9;
240 } else if (bvec_split_segs(q, &bv, &nsegs, §ors,
241 max_segs)) {
242 goto split;
243 }
244 }
245
246 *segs = nsegs;
247 return NULL;
248split:
249 *segs = nsegs;
250 return bio_split(bio, sectors, GFP_NOIO, bs);
251}
252
253void __blk_queue_split(struct request_queue *q, struct bio **bio,
254 unsigned int *nr_segs)
255{
256 struct bio *split;
257
258 switch (bio_op(*bio)) {
259 case REQ_OP_DISCARD:
260 case REQ_OP_SECURE_ERASE:
261 split = blk_bio_discard_split(q, *bio, &q->bio_split, nr_segs);
262 break;
263 case REQ_OP_WRITE_ZEROES:
264 split = blk_bio_write_zeroes_split(q, *bio, &q->bio_split,
265 nr_segs);
266 break;
267 case REQ_OP_WRITE_SAME:
268 split = blk_bio_write_same_split(q, *bio, &q->bio_split,
269 nr_segs);
270 break;
271 default:
272 split = blk_bio_segment_split(q, *bio, &q->bio_split, nr_segs);
273 break;
274 }
275
276 if (split) {
277
278 split->bi_opf |= REQ_NOMERGE;
279
280
281
282
283
284
285
286
287
288 bio_set_flag(*bio, BIO_QUEUE_ENTERED);
289
290 bio_chain(split, *bio);
291 trace_block_split(q, split, (*bio)->bi_iter.bi_sector);
292 generic_make_request(*bio);
293 *bio = split;
294 }
295}
296
297void blk_queue_split(struct request_queue *q, struct bio **bio)
298{
299 unsigned int nr_segs;
300
301 __blk_queue_split(q, bio, &nr_segs);
302}
303EXPORT_SYMBOL(blk_queue_split);
304
305unsigned int blk_recalc_rq_segments(struct request *rq)
306{
307 unsigned int nr_phys_segs = 0;
308 struct req_iterator iter;
309 struct bio_vec bv;
310
311 if (!rq->bio)
312 return 0;
313
314 switch (bio_op(rq->bio)) {
315 case REQ_OP_DISCARD:
316 case REQ_OP_SECURE_ERASE:
317 case REQ_OP_WRITE_ZEROES:
318 return 0;
319 case REQ_OP_WRITE_SAME:
320 return 1;
321 }
322
323 rq_for_each_bvec(bv, rq, iter)
324 bvec_split_segs(rq->q, &bv, &nr_phys_segs, NULL, UINT_MAX);
325 return nr_phys_segs;
326}
327
328static inline struct scatterlist *blk_next_sg(struct scatterlist **sg,
329 struct scatterlist *sglist)
330{
331 if (!*sg)
332 return sglist;
333
334
335
336
337
338
339
340
341 sg_unmark_end(*sg);
342 return sg_next(*sg);
343}
344
345static unsigned blk_bvec_map_sg(struct request_queue *q,
346 struct bio_vec *bvec, struct scatterlist *sglist,
347 struct scatterlist **sg)
348{
349 unsigned nbytes = bvec->bv_len;
350 unsigned nsegs = 0, total = 0;
351
352 while (nbytes > 0) {
353 unsigned offset = bvec->bv_offset + total;
354 unsigned len = min(get_max_segment_size(q, offset), nbytes);
355 struct page *page = bvec->bv_page;
356
357
358
359
360
361
362
363
364
365 page += (offset >> PAGE_SHIFT);
366 offset &= ~PAGE_MASK;
367
368 *sg = blk_next_sg(sg, sglist);
369 sg_set_page(*sg, page, len, offset);
370
371 total += len;
372 nbytes -= len;
373 nsegs++;
374 }
375
376 return nsegs;
377}
378
379static inline int __blk_bvec_map_sg(struct bio_vec bv,
380 struct scatterlist *sglist, struct scatterlist **sg)
381{
382 *sg = blk_next_sg(sg, sglist);
383 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
384 return 1;
385}
386
387
388static inline bool
389__blk_segment_map_sg_merge(struct request_queue *q, struct bio_vec *bvec,
390 struct bio_vec *bvprv, struct scatterlist **sg)
391{
392
393 int nbytes = bvec->bv_len;
394
395 if (!*sg)
396 return false;
397
398 if ((*sg)->length + nbytes > queue_max_segment_size(q))
399 return false;
400
401 if (!biovec_phys_mergeable(q, bvprv, bvec))
402 return false;
403
404 (*sg)->length += nbytes;
405
406 return true;
407}
408
409static int __blk_bios_map_sg(struct request_queue *q, struct bio *bio,
410 struct scatterlist *sglist,
411 struct scatterlist **sg)
412{
413 struct bio_vec uninitialized_var(bvec), bvprv = { NULL };
414 struct bvec_iter iter;
415 int nsegs = 0;
416 bool new_bio = false;
417
418 for_each_bio(bio) {
419 bio_for_each_bvec(bvec, bio, iter) {
420
421
422
423
424
425 if (new_bio &&
426 __blk_segment_map_sg_merge(q, &bvec, &bvprv, sg))
427 goto next_bvec;
428
429 if (bvec.bv_offset + bvec.bv_len <= PAGE_SIZE)
430 nsegs += __blk_bvec_map_sg(bvec, sglist, sg);
431 else
432 nsegs += blk_bvec_map_sg(q, &bvec, sglist, sg);
433 next_bvec:
434 new_bio = false;
435 }
436 if (likely(bio->bi_iter.bi_size)) {
437 bvprv = bvec;
438 new_bio = true;
439 }
440 }
441
442 return nsegs;
443}
444
445
446
447
448
449int blk_rq_map_sg(struct request_queue *q, struct request *rq,
450 struct scatterlist *sglist)
451{
452 struct scatterlist *sg = NULL;
453 int nsegs = 0;
454
455 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
456 nsegs = __blk_bvec_map_sg(rq->special_vec, sglist, &sg);
457 else if (rq->bio && bio_op(rq->bio) == REQ_OP_WRITE_SAME)
458 nsegs = __blk_bvec_map_sg(bio_iovec(rq->bio), sglist, &sg);
459 else if (rq->bio)
460 nsegs = __blk_bios_map_sg(q, rq->bio, sglist, &sg);
461
462 if (unlikely(rq->rq_flags & RQF_COPY_USER) &&
463 (blk_rq_bytes(rq) & q->dma_pad_mask)) {
464 unsigned int pad_len =
465 (q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1;
466
467 sg->length += pad_len;
468 rq->extra_len += pad_len;
469 }
470
471 if (q->dma_drain_size && q->dma_drain_needed(rq)) {
472 if (op_is_write(req_op(rq)))
473 memset(q->dma_drain_buffer, 0, q->dma_drain_size);
474
475 sg_unmark_end(sg);
476 sg = sg_next(sg);
477 sg_set_page(sg, virt_to_page(q->dma_drain_buffer),
478 q->dma_drain_size,
479 ((unsigned long)q->dma_drain_buffer) &
480 (PAGE_SIZE - 1));
481 nsegs++;
482 rq->extra_len += q->dma_drain_size;
483 }
484
485 if (sg)
486 sg_mark_end(sg);
487
488
489
490
491
492 WARN_ON(nsegs > blk_rq_nr_phys_segments(rq));
493
494 return nsegs;
495}
496EXPORT_SYMBOL(blk_rq_map_sg);
497
498static inline int ll_new_hw_segment(struct request *req, struct bio *bio,
499 unsigned int nr_phys_segs)
500{
501 if (req->nr_phys_segments + nr_phys_segs > queue_max_segments(req->q))
502 goto no_merge;
503
504 if (blk_integrity_merge_bio(req->q, req, bio) == false)
505 goto no_merge;
506
507
508
509
510
511 req->nr_phys_segments += nr_phys_segs;
512 return 1;
513
514no_merge:
515 req_set_nomerge(req->q, req);
516 return 0;
517}
518
519int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
520{
521 if (req_gap_back_merge(req, bio))
522 return 0;
523 if (blk_integrity_rq(req) &&
524 integrity_req_gap_back_merge(req, bio))
525 return 0;
526 if (blk_rq_sectors(req) + bio_sectors(bio) >
527 blk_rq_get_max_sectors(req, blk_rq_pos(req))) {
528 req_set_nomerge(req->q, req);
529 return 0;
530 }
531
532 return ll_new_hw_segment(req, bio, nr_segs);
533}
534
535int ll_front_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
536{
537 if (req_gap_front_merge(req, bio))
538 return 0;
539 if (blk_integrity_rq(req) &&
540 integrity_req_gap_front_merge(req, bio))
541 return 0;
542 if (blk_rq_sectors(req) + bio_sectors(bio) >
543 blk_rq_get_max_sectors(req, bio->bi_iter.bi_sector)) {
544 req_set_nomerge(req->q, req);
545 return 0;
546 }
547
548 return ll_new_hw_segment(req, bio, nr_segs);
549}
550
551static bool req_attempt_discard_merge(struct request_queue *q, struct request *req,
552 struct request *next)
553{
554 unsigned short segments = blk_rq_nr_discard_segments(req);
555
556 if (segments >= queue_max_discard_segments(q))
557 goto no_merge;
558 if (blk_rq_sectors(req) + bio_sectors(next->bio) >
559 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
560 goto no_merge;
561
562 req->nr_phys_segments = segments + blk_rq_nr_discard_segments(next);
563 return true;
564no_merge:
565 req_set_nomerge(q, req);
566 return false;
567}
568
569static int ll_merge_requests_fn(struct request_queue *q, struct request *req,
570 struct request *next)
571{
572 int total_phys_segments;
573
574 if (req_gap_back_merge(req, next->bio))
575 return 0;
576
577
578
579
580 if ((blk_rq_sectors(req) + blk_rq_sectors(next)) >
581 blk_rq_get_max_sectors(req, blk_rq_pos(req)))
582 return 0;
583
584 total_phys_segments = req->nr_phys_segments + next->nr_phys_segments;
585 if (total_phys_segments > queue_max_segments(q))
586 return 0;
587
588 if (blk_integrity_merge_rq(q, req, next) == false)
589 return 0;
590
591
592 req->nr_phys_segments = total_phys_segments;
593 return 1;
594}
595
596
597
598
599
600
601
602
603
604
605void blk_rq_set_mixed_merge(struct request *rq)
606{
607 unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
608 struct bio *bio;
609
610 if (rq->rq_flags & RQF_MIXED_MERGE)
611 return;
612
613
614
615
616
617
618 for (bio = rq->bio; bio; bio = bio->bi_next) {
619 WARN_ON_ONCE((bio->bi_opf & REQ_FAILFAST_MASK) &&
620 (bio->bi_opf & REQ_FAILFAST_MASK) != ff);
621 bio->bi_opf |= ff;
622 }
623 rq->rq_flags |= RQF_MIXED_MERGE;
624}
625
626static void blk_account_io_merge(struct request *req)
627{
628 if (blk_do_io_stat(req)) {
629 struct hd_struct *part;
630
631 part_stat_lock();
632 part = req->part;
633
634 part_dec_in_flight(req->q, part, rq_data_dir(req));
635
636 hd_struct_put(part);
637 part_stat_unlock();
638 }
639}
640
641
642
643
644
645
646
647
648static inline bool blk_discard_mergable(struct request *req)
649{
650 if (req_op(req) == REQ_OP_DISCARD &&
651 queue_max_discard_segments(req->q) > 1)
652 return true;
653 return false;
654}
655
656static enum elv_merge blk_try_req_merge(struct request *req,
657 struct request *next)
658{
659 if (blk_discard_mergable(req))
660 return ELEVATOR_DISCARD_MERGE;
661 else if (blk_rq_pos(req) + blk_rq_sectors(req) == blk_rq_pos(next))
662 return ELEVATOR_BACK_MERGE;
663
664 return ELEVATOR_NO_MERGE;
665}
666
667
668
669
670
671static struct request *attempt_merge(struct request_queue *q,
672 struct request *req, struct request *next)
673{
674 if (!rq_mergeable(req) || !rq_mergeable(next))
675 return NULL;
676
677 if (req_op(req) != req_op(next))
678 return NULL;
679
680 if (rq_data_dir(req) != rq_data_dir(next)
681 || req->rq_disk != next->rq_disk)
682 return NULL;
683
684 if (req_op(req) == REQ_OP_WRITE_SAME &&
685 !blk_write_same_mergeable(req->bio, next->bio))
686 return NULL;
687
688
689
690
691
692 if (req->write_hint != next->write_hint)
693 return NULL;
694
695 if (req->ioprio != next->ioprio)
696 return NULL;
697
698
699
700
701
702
703
704
705
706 switch (blk_try_req_merge(req, next)) {
707 case ELEVATOR_DISCARD_MERGE:
708 if (!req_attempt_discard_merge(q, req, next))
709 return NULL;
710 break;
711 case ELEVATOR_BACK_MERGE:
712 if (!ll_merge_requests_fn(q, req, next))
713 return NULL;
714 break;
715 default:
716 return NULL;
717 }
718
719
720
721
722
723
724
725 if (((req->rq_flags | next->rq_flags) & RQF_MIXED_MERGE) ||
726 (req->cmd_flags & REQ_FAILFAST_MASK) !=
727 (next->cmd_flags & REQ_FAILFAST_MASK)) {
728 blk_rq_set_mixed_merge(req);
729 blk_rq_set_mixed_merge(next);
730 }
731
732
733
734
735
736
737 if (next->start_time_ns < req->start_time_ns)
738 req->start_time_ns = next->start_time_ns;
739
740 req->biotail->bi_next = next->bio;
741 req->biotail = next->biotail;
742
743 req->__data_len += blk_rq_bytes(next);
744
745 if (!blk_discard_mergable(req))
746 elv_merge_requests(q, req, next);
747
748
749
750
751 blk_account_io_merge(next);
752
753
754
755
756
757 next->bio = NULL;
758 return next;
759}
760
761struct request *attempt_back_merge(struct request_queue *q, struct request *rq)
762{
763 struct request *next = elv_latter_request(q, rq);
764
765 if (next)
766 return attempt_merge(q, rq, next);
767
768 return NULL;
769}
770
771struct request *attempt_front_merge(struct request_queue *q, struct request *rq)
772{
773 struct request *prev = elv_former_request(q, rq);
774
775 if (prev)
776 return attempt_merge(q, prev, rq);
777
778 return NULL;
779}
780
781int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
782 struct request *next)
783{
784 struct request *free;
785
786 free = attempt_merge(q, rq, next);
787 if (free) {
788 blk_put_request(free);
789 return 1;
790 }
791
792 return 0;
793}
794
795bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
796{
797 if (!rq_mergeable(rq) || !bio_mergeable(bio))
798 return false;
799
800 if (req_op(rq) != bio_op(bio))
801 return false;
802
803
804 if (bio_data_dir(bio) != rq_data_dir(rq))
805 return false;
806
807
808 if (rq->rq_disk != bio->bi_disk)
809 return false;
810
811
812 if (blk_integrity_merge_bio(rq->q, rq, bio) == false)
813 return false;
814
815
816 if (req_op(rq) == REQ_OP_WRITE_SAME &&
817 !blk_write_same_mergeable(rq->bio, bio))
818 return false;
819
820
821
822
823
824 if (rq->write_hint != bio->bi_write_hint)
825 return false;
826
827 if (rq->ioprio != bio_prio(bio))
828 return false;
829
830 return true;
831}
832
833enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)
834{
835 if (blk_discard_mergable(rq))
836 return ELEVATOR_DISCARD_MERGE;
837 else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)
838 return ELEVATOR_BACK_MERGE;
839 else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector)
840 return ELEVATOR_FRONT_MERGE;
841 return ELEVATOR_NO_MERGE;
842}
843