1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/kernel.h>
16#include <linux/module.h>
17#include <linux/bio.h>
18#include <linux/blkdev.h>
19#include <linux/blk-pm.h>
20#include <linux/blk-integrity.h>
21#include <linux/highmem.h>
22#include <linux/mm.h>
23#include <linux/pagemap.h>
24#include <linux/kernel_stat.h>
25#include <linux/string.h>
26#include <linux/init.h>
27#include <linux/completion.h>
28#include <linux/slab.h>
29#include <linux/swap.h>
30#include <linux/writeback.h>
31#include <linux/task_io_accounting_ops.h>
32#include <linux/fault-inject.h>
33#include <linux/list_sort.h>
34#include <linux/delay.h>
35#include <linux/ratelimit.h>
36#include <linux/pm_runtime.h>
37#include <linux/t10-pi.h>
38#include <linux/debugfs.h>
39#include <linux/bpf.h>
40#include <linux/psi.h>
41#include <linux/part_stat.h>
42#include <linux/sched/sysctl.h>
43#include <linux/blk-crypto.h>
44
45#define CREATE_TRACE_POINTS
46#include <trace/events/block.h>
47
48#include "blk.h"
49#include "blk-mq-sched.h"
50#include "blk-pm.h"
51#include "blk-cgroup.h"
52#include "blk-throttle.h"
53
54struct dentry *blk_debugfs_root;
55
56EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
57EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
58EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
59EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
60EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
61EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_insert);
62
63DEFINE_IDA(blk_queue_ida);
64
65
66
67
68struct kmem_cache *blk_requestq_cachep;
69struct kmem_cache *blk_requestq_srcu_cachep;
70
71
72
73
74static struct workqueue_struct *kblockd_workqueue;
75
76
77
78
79
80
81void blk_queue_flag_set(unsigned int flag, struct request_queue *q)
82{
83 set_bit(flag, &q->queue_flags);
84}
85EXPORT_SYMBOL(blk_queue_flag_set);
86
87
88
89
90
91
92void blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
93{
94 clear_bit(flag, &q->queue_flags);
95}
96EXPORT_SYMBOL(blk_queue_flag_clear);
97
98
99
100
101
102
103
104
105
106bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q)
107{
108 return test_and_set_bit(flag, &q->queue_flags);
109}
110EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set);
111
112#define REQ_OP_NAME(name) [REQ_OP_##name] = #name
113static const char *const blk_op_name[] = {
114 REQ_OP_NAME(READ),
115 REQ_OP_NAME(WRITE),
116 REQ_OP_NAME(FLUSH),
117 REQ_OP_NAME(DISCARD),
118 REQ_OP_NAME(SECURE_ERASE),
119 REQ_OP_NAME(ZONE_RESET),
120 REQ_OP_NAME(ZONE_RESET_ALL),
121 REQ_OP_NAME(ZONE_OPEN),
122 REQ_OP_NAME(ZONE_CLOSE),
123 REQ_OP_NAME(ZONE_FINISH),
124 REQ_OP_NAME(ZONE_APPEND),
125 REQ_OP_NAME(WRITE_ZEROES),
126 REQ_OP_NAME(DRV_IN),
127 REQ_OP_NAME(DRV_OUT),
128};
129#undef REQ_OP_NAME
130
131
132
133
134
135
136
137
138
139inline const char *blk_op_str(unsigned int op)
140{
141 const char *op_str = "UNKNOWN";
142
143 if (op < ARRAY_SIZE(blk_op_name) && blk_op_name[op])
144 op_str = blk_op_name[op];
145
146 return op_str;
147}
148EXPORT_SYMBOL_GPL(blk_op_str);
149
150static const struct {
151 int errno;
152 const char *name;
153} blk_errors[] = {
154 [BLK_STS_OK] = { 0, "" },
155 [BLK_STS_NOTSUPP] = { -EOPNOTSUPP, "operation not supported" },
156 [BLK_STS_TIMEOUT] = { -ETIMEDOUT, "timeout" },
157 [BLK_STS_NOSPC] = { -ENOSPC, "critical space allocation" },
158 [BLK_STS_TRANSPORT] = { -ENOLINK, "recoverable transport" },
159 [BLK_STS_TARGET] = { -EREMOTEIO, "critical target" },
160 [BLK_STS_NEXUS] = { -EBADE, "critical nexus" },
161 [BLK_STS_MEDIUM] = { -ENODATA, "critical medium" },
162 [BLK_STS_PROTECTION] = { -EILSEQ, "protection" },
163 [BLK_STS_RESOURCE] = { -ENOMEM, "kernel resource" },
164 [BLK_STS_DEV_RESOURCE] = { -EBUSY, "device resource" },
165 [BLK_STS_AGAIN] = { -EAGAIN, "nonblocking retry" },
166 [BLK_STS_OFFLINE] = { -ENODEV, "device offline" },
167
168
169 [BLK_STS_DM_REQUEUE] = { -EREMCHG, "dm internal retry" },
170
171
172 [BLK_STS_ZONE_OPEN_RESOURCE] = { -ETOOMANYREFS, "open zones exceeded" },
173 [BLK_STS_ZONE_ACTIVE_RESOURCE] = { -EOVERFLOW, "active zones exceeded" },
174
175
176 [BLK_STS_IOERR] = { -EIO, "I/O" },
177};
178
179blk_status_t errno_to_blk_status(int errno)
180{
181 int i;
182
183 for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
184 if (blk_errors[i].errno == errno)
185 return (__force blk_status_t)i;
186 }
187
188 return BLK_STS_IOERR;
189}
190EXPORT_SYMBOL_GPL(errno_to_blk_status);
191
192int blk_status_to_errno(blk_status_t status)
193{
194 int idx = (__force int)status;
195
196 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
197 return -EIO;
198 return blk_errors[idx].errno;
199}
200EXPORT_SYMBOL_GPL(blk_status_to_errno);
201
202const char *blk_status_to_str(blk_status_t status)
203{
204 int idx = (__force int)status;
205
206 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
207 return "<null>";
208 return blk_errors[idx].name;
209}
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229void blk_sync_queue(struct request_queue *q)
230{
231 del_timer_sync(&q->timeout);
232 cancel_work_sync(&q->timeout_work);
233}
234EXPORT_SYMBOL(blk_sync_queue);
235
236
237
238
239
240void blk_set_pm_only(struct request_queue *q)
241{
242 atomic_inc(&q->pm_only);
243}
244EXPORT_SYMBOL_GPL(blk_set_pm_only);
245
246void blk_clear_pm_only(struct request_queue *q)
247{
248 int pm_only;
249
250 pm_only = atomic_dec_return(&q->pm_only);
251 WARN_ON_ONCE(pm_only < 0);
252 if (pm_only == 0)
253 wake_up_all(&q->mq_freeze_wq);
254}
255EXPORT_SYMBOL_GPL(blk_clear_pm_only);
256
257
258
259
260
261
262
263
264
265
266
267void blk_put_queue(struct request_queue *q)
268{
269 kobject_put(&q->kobj);
270}
271EXPORT_SYMBOL(blk_put_queue);
272
273void blk_queue_start_drain(struct request_queue *q)
274{
275
276
277
278
279
280 blk_freeze_queue_start(q);
281 if (queue_is_mq(q))
282 blk_mq_wake_waiters(q);
283
284 wake_up_all(&q->mq_freeze_wq);
285}
286
287
288
289
290
291
292
293
294
295
296void blk_cleanup_queue(struct request_queue *q)
297{
298
299 might_sleep();
300
301 WARN_ON_ONCE(blk_queue_registered(q));
302
303
304 blk_queue_flag_set(QUEUE_FLAG_DYING, q);
305 blk_queue_start_drain(q);
306
307 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
308 blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
309
310
311
312
313
314
315 blk_freeze_queue(q);
316
317 blk_queue_flag_set(QUEUE_FLAG_DEAD, q);
318
319 blk_sync_queue(q);
320 if (queue_is_mq(q)) {
321 blk_mq_cancel_work_sync(q);
322 blk_mq_exit_queue(q);
323 }
324
325
326
327
328
329
330
331
332
333 mutex_lock(&q->sysfs_lock);
334 if (q->elevator)
335 blk_mq_sched_free_rqs(q);
336 mutex_unlock(&q->sysfs_lock);
337
338
339 blk_put_queue(q);
340}
341EXPORT_SYMBOL(blk_cleanup_queue);
342
343
344
345
346
347
348int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
349{
350 const bool pm = flags & BLK_MQ_REQ_PM;
351
352 while (!blk_try_enter_queue(q, pm)) {
353 if (flags & BLK_MQ_REQ_NOWAIT)
354 return -EBUSY;
355
356
357
358
359
360
361
362
363 smp_rmb();
364 wait_event(q->mq_freeze_wq,
365 (!q->mq_freeze_depth &&
366 blk_pm_resume_queue(pm, q)) ||
367 blk_queue_dying(q));
368 if (blk_queue_dying(q))
369 return -ENODEV;
370 }
371
372 return 0;
373}
374
375int __bio_queue_enter(struct request_queue *q, struct bio *bio)
376{
377 while (!blk_try_enter_queue(q, false)) {
378 struct gendisk *disk = bio->bi_bdev->bd_disk;
379
380 if (bio->bi_opf & REQ_NOWAIT) {
381 if (test_bit(GD_DEAD, &disk->state))
382 goto dead;
383 bio_wouldblock_error(bio);
384 return -EBUSY;
385 }
386
387
388
389
390
391
392
393
394 smp_rmb();
395 wait_event(q->mq_freeze_wq,
396 (!q->mq_freeze_depth &&
397 blk_pm_resume_queue(false, q)) ||
398 test_bit(GD_DEAD, &disk->state));
399 if (test_bit(GD_DEAD, &disk->state))
400 goto dead;
401 }
402
403 return 0;
404dead:
405 bio_io_error(bio);
406 return -ENODEV;
407}
408
409void blk_queue_exit(struct request_queue *q)
410{
411 percpu_ref_put(&q->q_usage_counter);
412}
413
414static void blk_queue_usage_counter_release(struct percpu_ref *ref)
415{
416 struct request_queue *q =
417 container_of(ref, struct request_queue, q_usage_counter);
418
419 wake_up_all(&q->mq_freeze_wq);
420}
421
422static void blk_rq_timed_out_timer(struct timer_list *t)
423{
424 struct request_queue *q = from_timer(q, t, timeout);
425
426 kblockd_schedule_work(&q->timeout_work);
427}
428
429static void blk_timeout_work(struct work_struct *work)
430{
431}
432
433struct request_queue *blk_alloc_queue(int node_id, bool alloc_srcu)
434{
435 struct request_queue *q;
436 int ret;
437
438 q = kmem_cache_alloc_node(blk_get_queue_kmem_cache(alloc_srcu),
439 GFP_KERNEL | __GFP_ZERO, node_id);
440 if (!q)
441 return NULL;
442
443 if (alloc_srcu) {
444 blk_queue_flag_set(QUEUE_FLAG_HAS_SRCU, q);
445 if (init_srcu_struct(q->srcu) != 0)
446 goto fail_q;
447 }
448
449 q->last_merge = NULL;
450
451 q->id = ida_simple_get(&blk_queue_ida, 0, 0, GFP_KERNEL);
452 if (q->id < 0)
453 goto fail_srcu;
454
455 ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, 0);
456 if (ret)
457 goto fail_id;
458
459 q->stats = blk_alloc_queue_stats();
460 if (!q->stats)
461 goto fail_split;
462
463 q->node = node_id;
464
465 atomic_set(&q->nr_active_requests_shared_tags, 0);
466
467 timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
468 INIT_WORK(&q->timeout_work, blk_timeout_work);
469 INIT_LIST_HEAD(&q->icq_list);
470
471 kobject_init(&q->kobj, &blk_queue_ktype);
472
473 mutex_init(&q->debugfs_mutex);
474 mutex_init(&q->sysfs_lock);
475 mutex_init(&q->sysfs_dir_lock);
476 spin_lock_init(&q->queue_lock);
477
478 init_waitqueue_head(&q->mq_freeze_wq);
479 mutex_init(&q->mq_freeze_lock);
480
481
482
483
484
485 if (percpu_ref_init(&q->q_usage_counter,
486 blk_queue_usage_counter_release,
487 PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
488 goto fail_stats;
489
490 blk_queue_dma_alignment(q, 511);
491 blk_set_default_limits(&q->limits);
492 q->nr_requests = BLKDEV_DEFAULT_RQ;
493
494 return q;
495
496fail_stats:
497 blk_free_queue_stats(q->stats);
498fail_split:
499 bioset_exit(&q->bio_split);
500fail_id:
501 ida_simple_remove(&blk_queue_ida, q->id);
502fail_srcu:
503 if (alloc_srcu)
504 cleanup_srcu_struct(q->srcu);
505fail_q:
506 kmem_cache_free(blk_get_queue_kmem_cache(alloc_srcu), q);
507 return NULL;
508}
509
510
511
512
513
514
515
516
517
518bool blk_get_queue(struct request_queue *q)
519{
520 if (likely(!blk_queue_dying(q))) {
521 __blk_get_queue(q);
522 return true;
523 }
524
525 return false;
526}
527EXPORT_SYMBOL(blk_get_queue);
528
529#ifdef CONFIG_FAIL_MAKE_REQUEST
530
531static DECLARE_FAULT_ATTR(fail_make_request);
532
533static int __init setup_fail_make_request(char *str)
534{
535 return setup_fault_attr(&fail_make_request, str);
536}
537__setup("fail_make_request=", setup_fail_make_request);
538
539bool should_fail_request(struct block_device *part, unsigned int bytes)
540{
541 return part->bd_make_it_fail && should_fail(&fail_make_request, bytes);
542}
543
544static int __init fail_make_request_debugfs(void)
545{
546 struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
547 NULL, &fail_make_request);
548
549 return PTR_ERR_OR_ZERO(dir);
550}
551
552late_initcall(fail_make_request_debugfs);
553#endif
554
555static inline bool bio_check_ro(struct bio *bio)
556{
557 if (op_is_write(bio_op(bio)) && bdev_read_only(bio->bi_bdev)) {
558 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
559 return false;
560 pr_warn("Trying to write to read-only block-device %pg\n",
561 bio->bi_bdev);
562
563 return false;
564 }
565
566 return false;
567}
568
569static noinline int should_fail_bio(struct bio *bio)
570{
571 if (should_fail_request(bdev_whole(bio->bi_bdev), bio->bi_iter.bi_size))
572 return -EIO;
573 return 0;
574}
575ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);
576
577
578
579
580
581
582static inline int bio_check_eod(struct bio *bio)
583{
584 sector_t maxsector = bdev_nr_sectors(bio->bi_bdev);
585 unsigned int nr_sectors = bio_sectors(bio);
586
587 if (nr_sectors && maxsector &&
588 (nr_sectors > maxsector ||
589 bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
590 pr_info_ratelimited("%s: attempt to access beyond end of device\n"
591 "%pg: rw=%d, want=%llu, limit=%llu\n",
592 current->comm,
593 bio->bi_bdev, bio->bi_opf,
594 bio_end_sector(bio), maxsector);
595 return -EIO;
596 }
597 return 0;
598}
599
600
601
602
603static int blk_partition_remap(struct bio *bio)
604{
605 struct block_device *p = bio->bi_bdev;
606
607 if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
608 return -EIO;
609 if (bio_sectors(bio)) {
610 bio->bi_iter.bi_sector += p->bd_start_sect;
611 trace_block_bio_remap(bio, p->bd_dev,
612 bio->bi_iter.bi_sector -
613 p->bd_start_sect);
614 }
615 bio_set_flag(bio, BIO_REMAPPED);
616 return 0;
617}
618
619
620
621
622static inline blk_status_t blk_check_zone_append(struct request_queue *q,
623 struct bio *bio)
624{
625 sector_t pos = bio->bi_iter.bi_sector;
626 int nr_sectors = bio_sectors(bio);
627
628
629 if (!blk_queue_is_zoned(q))
630 return BLK_STS_NOTSUPP;
631
632
633 if (pos & (blk_queue_zone_sectors(q) - 1) ||
634 !blk_queue_zone_is_seq(q, pos))
635 return BLK_STS_IOERR;
636
637
638
639
640
641
642 if (nr_sectors > q->limits.chunk_sectors)
643 return BLK_STS_IOERR;
644
645
646 if (nr_sectors > q->limits.max_zone_append_sectors)
647 return BLK_STS_IOERR;
648
649 bio->bi_opf |= REQ_NOMERGE;
650
651 return BLK_STS_OK;
652}
653
654static void __submit_bio(struct bio *bio)
655{
656 struct gendisk *disk = bio->bi_bdev->bd_disk;
657
658 if (unlikely(!blk_crypto_bio_prep(&bio)))
659 return;
660
661 if (!disk->fops->submit_bio) {
662 blk_mq_submit_bio(bio);
663 } else if (likely(bio_queue_enter(bio) == 0)) {
664 disk->fops->submit_bio(bio);
665 blk_queue_exit(disk->queue);
666 }
667}
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688static void __submit_bio_noacct(struct bio *bio)
689{
690 struct bio_list bio_list_on_stack[2];
691
692 BUG_ON(bio->bi_next);
693
694 bio_list_init(&bio_list_on_stack[0]);
695 current->bio_list = bio_list_on_stack;
696
697 do {
698 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
699 struct bio_list lower, same;
700
701
702
703
704 bio_list_on_stack[1] = bio_list_on_stack[0];
705 bio_list_init(&bio_list_on_stack[0]);
706
707 __submit_bio(bio);
708
709
710
711
712
713 bio_list_init(&lower);
714 bio_list_init(&same);
715 while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
716 if (q == bdev_get_queue(bio->bi_bdev))
717 bio_list_add(&same, bio);
718 else
719 bio_list_add(&lower, bio);
720
721
722
723
724 bio_list_merge(&bio_list_on_stack[0], &lower);
725 bio_list_merge(&bio_list_on_stack[0], &same);
726 bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
727 } while ((bio = bio_list_pop(&bio_list_on_stack[0])));
728
729 current->bio_list = NULL;
730}
731
732static void __submit_bio_noacct_mq(struct bio *bio)
733{
734 struct bio_list bio_list[2] = { };
735
736 current->bio_list = bio_list;
737
738 do {
739 __submit_bio(bio);
740 } while ((bio = bio_list_pop(&bio_list[0])));
741
742 current->bio_list = NULL;
743}
744
745void submit_bio_noacct_nocheck(struct bio *bio)
746{
747
748
749
750
751
752
753 if (current->bio_list)
754 bio_list_add(¤t->bio_list[0], bio);
755 else if (!bio->bi_bdev->bd_disk->fops->submit_bio)
756 __submit_bio_noacct_mq(bio);
757 else
758 __submit_bio_noacct(bio);
759}
760
761
762
763
764
765
766
767
768
769
770void submit_bio_noacct(struct bio *bio)
771{
772 struct block_device *bdev = bio->bi_bdev;
773 struct request_queue *q = bdev_get_queue(bdev);
774 blk_status_t status = BLK_STS_IOERR;
775 struct blk_plug *plug;
776
777 might_sleep();
778
779 plug = blk_mq_plug(q, bio);
780 if (plug && plug->nowait)
781 bio->bi_opf |= REQ_NOWAIT;
782
783
784
785
786
787 if ((bio->bi_opf & REQ_NOWAIT) && !blk_queue_nowait(q))
788 goto not_supported;
789
790 if (should_fail_bio(bio))
791 goto end_io;
792 if (unlikely(bio_check_ro(bio)))
793 goto end_io;
794 if (!bio_flagged(bio, BIO_REMAPPED)) {
795 if (unlikely(bio_check_eod(bio)))
796 goto end_io;
797 if (bdev->bd_partno && unlikely(blk_partition_remap(bio)))
798 goto end_io;
799 }
800
801
802
803
804
805 if (op_is_flush(bio->bi_opf) &&
806 !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
807 bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
808 if (!bio_sectors(bio)) {
809 status = BLK_STS_OK;
810 goto end_io;
811 }
812 }
813
814 if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
815 bio_clear_polled(bio);
816
817 switch (bio_op(bio)) {
818 case REQ_OP_DISCARD:
819 if (!blk_queue_discard(q))
820 goto not_supported;
821 break;
822 case REQ_OP_SECURE_ERASE:
823 if (!blk_queue_secure_erase(q))
824 goto not_supported;
825 break;
826 case REQ_OP_ZONE_APPEND:
827 status = blk_check_zone_append(q, bio);
828 if (status != BLK_STS_OK)
829 goto end_io;
830 break;
831 case REQ_OP_ZONE_RESET:
832 case REQ_OP_ZONE_OPEN:
833 case REQ_OP_ZONE_CLOSE:
834 case REQ_OP_ZONE_FINISH:
835 if (!blk_queue_is_zoned(q))
836 goto not_supported;
837 break;
838 case REQ_OP_ZONE_RESET_ALL:
839 if (!blk_queue_is_zoned(q) || !blk_queue_zone_resetall(q))
840 goto not_supported;
841 break;
842 case REQ_OP_WRITE_ZEROES:
843 if (!q->limits.max_write_zeroes_sectors)
844 goto not_supported;
845 break;
846 default:
847 break;
848 }
849
850 if (blk_throtl_bio(bio))
851 return;
852
853 blk_cgroup_bio_start(bio);
854 blkcg_bio_issue_init(bio);
855
856 if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
857 trace_block_bio_queue(bio);
858
859
860
861 bio_set_flag(bio, BIO_TRACE_COMPLETION);
862 }
863 submit_bio_noacct_nocheck(bio);
864 return;
865
866not_supported:
867 status = BLK_STS_NOTSUPP;
868end_io:
869 bio->bi_status = status;
870 bio_endio(bio);
871}
872EXPORT_SYMBOL(submit_bio_noacct);
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887void submit_bio(struct bio *bio)
888{
889 if (blkcg_punt_bio_submit(bio))
890 return;
891
892
893
894
895
896 if (bio_has_data(bio)) {
897 unsigned int count = bio_sectors(bio);
898
899 if (op_is_write(bio_op(bio))) {
900 count_vm_events(PGPGOUT, count);
901 } else {
902 task_io_account_read(bio->bi_iter.bi_size);
903 count_vm_events(PGPGIN, count);
904 }
905 }
906
907
908
909
910
911
912
913 if (unlikely(bio_op(bio) == REQ_OP_READ &&
914 bio_flagged(bio, BIO_WORKINGSET))) {
915 unsigned long pflags;
916
917 psi_memstall_enter(&pflags);
918 submit_bio_noacct(bio);
919 psi_memstall_leave(&pflags);
920 return;
921 }
922
923 submit_bio_noacct(bio);
924}
925EXPORT_SYMBOL(submit_bio);
926
927
928
929
930
931
932
933
934
935
936
937
938
939int bio_poll(struct bio *bio, struct io_comp_batch *iob, unsigned int flags)
940{
941 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
942 blk_qc_t cookie = READ_ONCE(bio->bi_cookie);
943 int ret = 0;
944
945 if (cookie == BLK_QC_T_NONE ||
946 !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
947 return 0;
948
949 blk_flush_plug(current->plug, false);
950
951 if (blk_queue_enter(q, BLK_MQ_REQ_NOWAIT))
952 return 0;
953 if (queue_is_mq(q)) {
954 ret = blk_mq_poll(q, cookie, iob, flags);
955 } else {
956 struct gendisk *disk = q->disk;
957
958 if (disk && disk->fops->poll_bio)
959 ret = disk->fops->poll_bio(bio, iob, flags);
960 }
961 blk_queue_exit(q);
962 return ret;
963}
964EXPORT_SYMBOL_GPL(bio_poll);
965
966
967
968
969
970int iocb_bio_iopoll(struct kiocb *kiocb, struct io_comp_batch *iob,
971 unsigned int flags)
972{
973 struct bio *bio;
974 int ret = 0;
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996 rcu_read_lock();
997 bio = READ_ONCE(kiocb->private);
998 if (bio && bio->bi_bdev)
999 ret = bio_poll(bio, iob, flags);
1000 rcu_read_unlock();
1001
1002 return ret;
1003}
1004EXPORT_SYMBOL_GPL(iocb_bio_iopoll);
1005
1006void update_io_ticks(struct block_device *part, unsigned long now, bool end)
1007{
1008 unsigned long stamp;
1009again:
1010 stamp = READ_ONCE(part->bd_stamp);
1011 if (unlikely(time_after(now, stamp))) {
1012 if (likely(cmpxchg(&part->bd_stamp, stamp, now) == stamp))
1013 __part_stat_add(part, io_ticks, end ? now - stamp : 1);
1014 }
1015 if (part->bd_partno) {
1016 part = bdev_whole(part);
1017 goto again;
1018 }
1019}
1020
1021static unsigned long __part_start_io_acct(struct block_device *part,
1022 unsigned int sectors, unsigned int op,
1023 unsigned long start_time)
1024{
1025 const int sgrp = op_stat_group(op);
1026
1027 part_stat_lock();
1028 update_io_ticks(part, start_time, false);
1029 part_stat_inc(part, ios[sgrp]);
1030 part_stat_add(part, sectors[sgrp], sectors);
1031 part_stat_local_inc(part, in_flight[op_is_write(op)]);
1032 part_stat_unlock();
1033
1034 return start_time;
1035}
1036
1037
1038
1039
1040
1041
1042void bio_start_io_acct_time(struct bio *bio, unsigned long start_time)
1043{
1044 __part_start_io_acct(bio->bi_bdev, bio_sectors(bio),
1045 bio_op(bio), start_time);
1046}
1047EXPORT_SYMBOL_GPL(bio_start_io_acct_time);
1048
1049
1050
1051
1052
1053
1054
1055unsigned long bio_start_io_acct(struct bio *bio)
1056{
1057 return __part_start_io_acct(bio->bi_bdev, bio_sectors(bio),
1058 bio_op(bio), jiffies);
1059}
1060EXPORT_SYMBOL_GPL(bio_start_io_acct);
1061
1062unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
1063 unsigned int op)
1064{
1065 return __part_start_io_acct(disk->part0, sectors, op, jiffies);
1066}
1067EXPORT_SYMBOL(disk_start_io_acct);
1068
1069static void __part_end_io_acct(struct block_device *part, unsigned int op,
1070 unsigned long start_time)
1071{
1072 const int sgrp = op_stat_group(op);
1073 unsigned long now = READ_ONCE(jiffies);
1074 unsigned long duration = now - start_time;
1075
1076 part_stat_lock();
1077 update_io_ticks(part, now, true);
1078 part_stat_add(part, nsecs[sgrp], jiffies_to_nsecs(duration));
1079 part_stat_local_dec(part, in_flight[op_is_write(op)]);
1080 part_stat_unlock();
1081}
1082
1083void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
1084 struct block_device *orig_bdev)
1085{
1086 __part_end_io_acct(orig_bdev, bio_op(bio), start_time);
1087}
1088EXPORT_SYMBOL_GPL(bio_end_io_acct_remapped);
1089
1090void disk_end_io_acct(struct gendisk *disk, unsigned int op,
1091 unsigned long start_time)
1092{
1093 __part_end_io_acct(disk->part0, op, start_time);
1094}
1095EXPORT_SYMBOL(disk_end_io_acct);
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116int blk_lld_busy(struct request_queue *q)
1117{
1118 if (queue_is_mq(q) && q->mq_ops->busy)
1119 return q->mq_ops->busy(q);
1120
1121 return 0;
1122}
1123EXPORT_SYMBOL_GPL(blk_lld_busy);
1124
1125int kblockd_schedule_work(struct work_struct *work)
1126{
1127 return queue_work(kblockd_workqueue, work);
1128}
1129EXPORT_SYMBOL(kblockd_schedule_work);
1130
1131int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
1132 unsigned long delay)
1133{
1134 return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
1135}
1136EXPORT_SYMBOL(kblockd_mod_delayed_work_on);
1137
1138void blk_start_plug_nr_ios(struct blk_plug *plug, unsigned short nr_ios)
1139{
1140 struct task_struct *tsk = current;
1141
1142
1143
1144
1145 if (tsk->plug)
1146 return;
1147
1148 plug->mq_list = NULL;
1149 plug->cached_rq = NULL;
1150 plug->nr_ios = min_t(unsigned short, nr_ios, BLK_MAX_REQUEST_COUNT);
1151 plug->rq_count = 0;
1152 plug->multiple_queues = false;
1153 plug->has_elevator = false;
1154 plug->nowait = false;
1155 INIT_LIST_HEAD(&plug->cb_list);
1156
1157
1158
1159
1160
1161 tsk->plug = plug;
1162}
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187void blk_start_plug(struct blk_plug *plug)
1188{
1189 blk_start_plug_nr_ios(plug, 1);
1190}
1191EXPORT_SYMBOL(blk_start_plug);
1192
1193static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
1194{
1195 LIST_HEAD(callbacks);
1196
1197 while (!list_empty(&plug->cb_list)) {
1198 list_splice_init(&plug->cb_list, &callbacks);
1199
1200 while (!list_empty(&callbacks)) {
1201 struct blk_plug_cb *cb = list_first_entry(&callbacks,
1202 struct blk_plug_cb,
1203 list);
1204 list_del(&cb->list);
1205 cb->callback(cb, from_schedule);
1206 }
1207 }
1208}
1209
1210struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
1211 int size)
1212{
1213 struct blk_plug *plug = current->plug;
1214 struct blk_plug_cb *cb;
1215
1216 if (!plug)
1217 return NULL;
1218
1219 list_for_each_entry(cb, &plug->cb_list, list)
1220 if (cb->callback == unplug && cb->data == data)
1221 return cb;
1222
1223
1224 BUG_ON(size < sizeof(*cb));
1225 cb = kzalloc(size, GFP_ATOMIC);
1226 if (cb) {
1227 cb->data = data;
1228 cb->callback = unplug;
1229 list_add(&cb->list, &plug->cb_list);
1230 }
1231 return cb;
1232}
1233EXPORT_SYMBOL(blk_check_plugged);
1234
1235void __blk_flush_plug(struct blk_plug *plug, bool from_schedule)
1236{
1237 if (!list_empty(&plug->cb_list))
1238 flush_plug_callbacks(plug, from_schedule);
1239 if (!rq_list_empty(plug->mq_list))
1240 blk_mq_flush_plug_list(plug, from_schedule);
1241
1242
1243
1244
1245
1246
1247 if (unlikely(!rq_list_empty(plug->cached_rq)))
1248 blk_mq_free_plug_rqs(plug);
1249}
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261void blk_finish_plug(struct blk_plug *plug)
1262{
1263 if (plug == current->plug) {
1264 __blk_flush_plug(plug, false);
1265 current->plug = NULL;
1266 }
1267}
1268EXPORT_SYMBOL(blk_finish_plug);
1269
1270void blk_io_schedule(void)
1271{
1272
1273 unsigned long timeout = sysctl_hung_task_timeout_secs * HZ / 2;
1274
1275 if (timeout)
1276 io_schedule_timeout(timeout);
1277 else
1278 io_schedule();
1279}
1280EXPORT_SYMBOL_GPL(blk_io_schedule);
1281
1282int __init blk_dev_init(void)
1283{
1284 BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
1285 BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
1286 sizeof_field(struct request, cmd_flags));
1287 BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
1288 sizeof_field(struct bio, bi_opf));
1289 BUILD_BUG_ON(ALIGN(offsetof(struct request_queue, srcu),
1290 __alignof__(struct request_queue)) !=
1291 sizeof(struct request_queue));
1292
1293
1294 kblockd_workqueue = alloc_workqueue("kblockd",
1295 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
1296 if (!kblockd_workqueue)
1297 panic("Failed to create kblockd\n");
1298
1299 blk_requestq_cachep = kmem_cache_create("request_queue",
1300 sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
1301
1302 blk_requestq_srcu_cachep = kmem_cache_create("request_queue_srcu",
1303 sizeof(struct request_queue) +
1304 sizeof(struct srcu_struct), 0, SLAB_PANIC, NULL);
1305
1306 blk_debugfs_root = debugfs_create_dir("block", NULL);
1307
1308 return 0;
1309}
1310