1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/backing-dev.h>
17#include <linux/bio.h>
18#include <linux/blkdev.h>
19#include <linux/highmem.h>
20#include <linux/mm.h>
21#include <linux/kernel_stat.h>
22#include <linux/string.h>
23#include <linux/init.h>
24#include <linux/completion.h>
25#include <linux/slab.h>
26#include <linux/swap.h>
27#include <linux/writeback.h>
28#include <linux/task_io_accounting_ops.h>
29#include <linux/fault-inject.h>
30#include <linux/list_sort.h>
31#include <linux/delay.h>
32#include <linux/ratelimit.h>
33#include <linux/pm_runtime.h>
34
35#define CREATE_TRACE_POINTS
36#include <trace/events/block.h>
37
38#include "blk.h"
39#include "blk-cgroup.h"
40
41EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
42EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
43EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
44EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
45
46DEFINE_IDA(blk_queue_ida);
47
48
49
50
51static struct kmem_cache *request_cachep;
52
53
54
55
56struct kmem_cache *blk_requestq_cachep;
57
58
59
60
61static struct workqueue_struct *kblockd_workqueue;
62
63static void drive_stat_acct(struct request *rq, int new_io)
64{
65 struct hd_struct *part;
66 int rw = rq_data_dir(rq);
67 int cpu;
68
69 if (!blk_do_io_stat(rq))
70 return;
71
72 cpu = part_stat_lock();
73
74 if (!new_io) {
75 part = rq->part;
76 part_stat_inc(cpu, part, merges[rw]);
77 } else {
78 part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
79 if (!hd_struct_try_get(part)) {
80
81
82
83
84
85
86
87
88 part = &rq->rq_disk->part0;
89 hd_struct_get(part);
90 }
91 part_round_stats(cpu, part);
92 part_inc_in_flight(part, rw);
93 rq->part = part;
94 }
95
96 part_stat_unlock();
97}
98
99void blk_queue_congestion_threshold(struct request_queue *q)
100{
101 int nr;
102
103 nr = q->nr_requests - (q->nr_requests / 8) + 1;
104 if (nr > q->nr_requests)
105 nr = q->nr_requests;
106 q->nr_congestion_on = nr;
107
108 nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
109 if (nr < 1)
110 nr = 1;
111 q->nr_congestion_off = nr;
112}
113
114
115
116
117
118
119
120
121
122
123struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
124{
125 struct backing_dev_info *ret = NULL;
126 struct request_queue *q = bdev_get_queue(bdev);
127
128 if (q)
129 ret = &q->backing_dev_info;
130 return ret;
131}
132EXPORT_SYMBOL(blk_get_backing_dev_info);
133
134void blk_rq_init(struct request_queue *q, struct request *rq)
135{
136 memset(rq, 0, sizeof(*rq));
137
138 INIT_LIST_HEAD(&rq->queuelist);
139 INIT_LIST_HEAD(&rq->timeout_list);
140 rq->cpu = -1;
141 rq->q = q;
142 rq->__sector = (sector_t) -1;
143 INIT_HLIST_NODE(&rq->hash);
144 RB_CLEAR_NODE(&rq->rb_node);
145 rq->cmd = rq->__cmd;
146 rq->cmd_len = BLK_MAX_CDB;
147 rq->tag = -1;
148 rq->ref_count = 1;
149 rq->start_time = jiffies;
150 set_start_time_ns(rq);
151 rq->part = NULL;
152}
153EXPORT_SYMBOL(blk_rq_init);
154
155static void req_bio_endio(struct request *rq, struct bio *bio,
156 unsigned int nbytes, int error)
157{
158 if (error)
159 clear_bit(BIO_UPTODATE, &bio->bi_flags);
160 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
161 error = -EIO;
162
163 if (unlikely(rq->cmd_flags & REQ_QUIET))
164 set_bit(BIO_QUIET, &bio->bi_flags);
165
166 bio_advance(bio, nbytes);
167
168
169 if (bio->bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ))
170 bio_endio(bio, error);
171}
172
173void blk_dump_rq_flags(struct request *rq, char *msg)
174{
175 int bit;
176
177 printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg,
178 rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
179 rq->cmd_flags);
180
181 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
182 (unsigned long long)blk_rq_pos(rq),
183 blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
184 printk(KERN_INFO " bio %p, biotail %p, buffer %p, len %u\n",
185 rq->bio, rq->biotail, rq->buffer, blk_rq_bytes(rq));
186
187 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
188 printk(KERN_INFO " cdb: ");
189 for (bit = 0; bit < BLK_MAX_CDB; bit++)
190 printk("%02x ", rq->cmd[bit]);
191 printk("\n");
192 }
193}
194EXPORT_SYMBOL(blk_dump_rq_flags);
195
196static void blk_delay_work(struct work_struct *work)
197{
198 struct request_queue *q;
199
200 q = container_of(work, struct request_queue, delay_work.work);
201 spin_lock_irq(q->queue_lock);
202 __blk_run_queue(q);
203 spin_unlock_irq(q->queue_lock);
204}
205
206
207
208
209
210
211
212
213
214
215
216void blk_delay_queue(struct request_queue *q, unsigned long msecs)
217{
218 if (likely(!blk_queue_dead(q)))
219 queue_delayed_work(kblockd_workqueue, &q->delay_work,
220 msecs_to_jiffies(msecs));
221}
222EXPORT_SYMBOL(blk_delay_queue);
223
224
225
226
227
228
229
230
231
232
233void blk_start_queue(struct request_queue *q)
234{
235 WARN_ON(!irqs_disabled());
236
237 queue_flag_clear(QUEUE_FLAG_STOPPED, q);
238 __blk_run_queue(q);
239}
240EXPORT_SYMBOL(blk_start_queue);
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256void blk_stop_queue(struct request_queue *q)
257{
258 cancel_delayed_work(&q->delay_work);
259 queue_flag_set(QUEUE_FLAG_STOPPED, q);
260}
261EXPORT_SYMBOL(blk_stop_queue);
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281void blk_sync_queue(struct request_queue *q)
282{
283 del_timer_sync(&q->timeout);
284 cancel_delayed_work_sync(&q->delay_work);
285}
286EXPORT_SYMBOL(blk_sync_queue);
287
288
289
290
291
292
293
294
295
296
297
298
299inline void __blk_run_queue_uncond(struct request_queue *q)
300{
301 if (unlikely(blk_queue_dead(q)))
302 return;
303
304
305
306
307
308
309
310
311 q->request_fn_active++;
312 q->request_fn(q);
313 q->request_fn_active--;
314}
315
316
317
318
319
320
321
322
323
324void __blk_run_queue(struct request_queue *q)
325{
326 if (unlikely(blk_queue_stopped(q)))
327 return;
328
329 __blk_run_queue_uncond(q);
330}
331EXPORT_SYMBOL(__blk_run_queue);
332
333
334
335
336
337
338
339
340
341void blk_run_queue_async(struct request_queue *q)
342{
343 if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q)))
344 mod_delayed_work(kblockd_workqueue, &q->delay_work, 0);
345}
346EXPORT_SYMBOL(blk_run_queue_async);
347
348
349
350
351
352
353
354
355
356void blk_run_queue(struct request_queue *q)
357{
358 unsigned long flags;
359
360 spin_lock_irqsave(q->queue_lock, flags);
361 __blk_run_queue(q);
362 spin_unlock_irqrestore(q->queue_lock, flags);
363}
364EXPORT_SYMBOL(blk_run_queue);
365
366void blk_put_queue(struct request_queue *q)
367{
368 kobject_put(&q->kobj);
369}
370EXPORT_SYMBOL(blk_put_queue);
371
372
373
374
375
376
377
378
379
380
381static void __blk_drain_queue(struct request_queue *q, bool drain_all)
382 __releases(q->queue_lock)
383 __acquires(q->queue_lock)
384{
385 int i;
386
387 lockdep_assert_held(q->queue_lock);
388
389 while (true) {
390 bool drain = false;
391
392
393
394
395
396 if (q->elevator)
397 elv_drain_elevator(q);
398
399 blkcg_drain_queue(q);
400
401
402
403
404
405
406
407
408 if (!list_empty(&q->queue_head) && q->request_fn)
409 __blk_run_queue(q);
410
411 drain |= q->nr_rqs_elvpriv;
412 drain |= q->request_fn_active;
413
414
415
416
417
418
419 if (drain_all) {
420 drain |= !list_empty(&q->queue_head);
421 for (i = 0; i < 2; i++) {
422 drain |= q->nr_rqs[i];
423 drain |= q->in_flight[i];
424 drain |= !list_empty(&q->flush_queue[i]);
425 }
426 }
427
428 if (!drain)
429 break;
430
431 spin_unlock_irq(q->queue_lock);
432
433 msleep(10);
434
435 spin_lock_irq(q->queue_lock);
436 }
437
438
439
440
441
442
443 if (q->request_fn) {
444 struct request_list *rl;
445
446 blk_queue_for_each_rl(rl, q)
447 for (i = 0; i < ARRAY_SIZE(rl->wait); i++)
448 wake_up_all(&rl->wait[i]);
449 }
450}
451
452
453
454
455
456
457
458
459
460
461
462void blk_queue_bypass_start(struct request_queue *q)
463{
464 bool drain;
465
466 spin_lock_irq(q->queue_lock);
467 drain = !q->bypass_depth++;
468 queue_flag_set(QUEUE_FLAG_BYPASS, q);
469 spin_unlock_irq(q->queue_lock);
470
471 if (drain) {
472 spin_lock_irq(q->queue_lock);
473 __blk_drain_queue(q, false);
474 spin_unlock_irq(q->queue_lock);
475
476
477 synchronize_rcu();
478 }
479}
480EXPORT_SYMBOL_GPL(blk_queue_bypass_start);
481
482
483
484
485
486
487
488void blk_queue_bypass_end(struct request_queue *q)
489{
490 spin_lock_irq(q->queue_lock);
491 if (!--q->bypass_depth)
492 queue_flag_clear(QUEUE_FLAG_BYPASS, q);
493 WARN_ON_ONCE(q->bypass_depth < 0);
494 spin_unlock_irq(q->queue_lock);
495}
496EXPORT_SYMBOL_GPL(blk_queue_bypass_end);
497
498
499
500
501
502
503
504
505void blk_cleanup_queue(struct request_queue *q)
506{
507 spinlock_t *lock = q->queue_lock;
508
509
510 mutex_lock(&q->sysfs_lock);
511 queue_flag_set_unlocked(QUEUE_FLAG_DYING, q);
512 spin_lock_irq(lock);
513
514
515
516
517
518
519
520
521
522
523 q->bypass_depth++;
524 queue_flag_set(QUEUE_FLAG_BYPASS, q);
525
526 queue_flag_set(QUEUE_FLAG_NOMERGES, q);
527 queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
528 queue_flag_set(QUEUE_FLAG_DYING, q);
529 spin_unlock_irq(lock);
530 mutex_unlock(&q->sysfs_lock);
531
532
533
534
535
536 spin_lock_irq(lock);
537 __blk_drain_queue(q, true);
538 queue_flag_set(QUEUE_FLAG_DEAD, q);
539 spin_unlock_irq(lock);
540
541
542 del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer);
543 blk_sync_queue(q);
544
545 spin_lock_irq(lock);
546 if (q->queue_lock != &q->__queue_lock)
547 q->queue_lock = &q->__queue_lock;
548 spin_unlock_irq(lock);
549
550
551 blk_put_queue(q);
552}
553EXPORT_SYMBOL(blk_cleanup_queue);
554
555int blk_init_rl(struct request_list *rl, struct request_queue *q,
556 gfp_t gfp_mask)
557{
558 if (unlikely(rl->rq_pool))
559 return 0;
560
561 rl->q = q;
562 rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
563 rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
564 init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
565 init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
566
567 rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
568 mempool_free_slab, request_cachep,
569 gfp_mask, q->node);
570 if (!rl->rq_pool)
571 return -ENOMEM;
572
573 return 0;
574}
575
576void blk_exit_rl(struct request_list *rl)
577{
578 if (rl->rq_pool)
579 mempool_destroy(rl->rq_pool);
580}
581
582struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
583{
584 return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE);
585}
586EXPORT_SYMBOL(blk_alloc_queue);
587
588struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
589{
590 struct request_queue *q;
591 int err;
592
593 q = kmem_cache_alloc_node(blk_requestq_cachep,
594 gfp_mask | __GFP_ZERO, node_id);
595 if (!q)
596 return NULL;
597
598 q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
599 if (q->id < 0)
600 goto fail_q;
601
602 q->backing_dev_info.ra_pages =
603 (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
604 q->backing_dev_info.state = 0;
605 q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY;
606 q->backing_dev_info.name = "block";
607 q->node = node_id;
608
609 err = bdi_init(&q->backing_dev_info);
610 if (err)
611 goto fail_id;
612
613 setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
614 laptop_mode_timer_fn, (unsigned long) q);
615 setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
616 INIT_LIST_HEAD(&q->queue_head);
617 INIT_LIST_HEAD(&q->timeout_list);
618 INIT_LIST_HEAD(&q->icq_list);
619#ifdef CONFIG_BLK_CGROUP
620 INIT_LIST_HEAD(&q->blkg_list);
621#endif
622 INIT_LIST_HEAD(&q->flush_queue[0]);
623 INIT_LIST_HEAD(&q->flush_queue[1]);
624 INIT_LIST_HEAD(&q->flush_data_in_flight);
625 INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
626
627 kobject_init(&q->kobj, &blk_queue_ktype);
628
629 mutex_init(&q->sysfs_lock);
630 spin_lock_init(&q->__queue_lock);
631
632
633
634
635
636 q->queue_lock = &q->__queue_lock;
637
638
639
640
641
642
643
644 q->bypass_depth = 1;
645 __set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
646
647 if (blkcg_init_queue(q))
648 goto fail_id;
649
650 return q;
651
652fail_id:
653 ida_simple_remove(&blk_queue_ida, q->id);
654fail_q:
655 kmem_cache_free(blk_requestq_cachep, q);
656 return NULL;
657}
658EXPORT_SYMBOL(blk_alloc_queue_node);
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
694{
695 return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
696}
697EXPORT_SYMBOL(blk_init_queue);
698
699struct request_queue *
700blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
701{
702 struct request_queue *uninit_q, *q;
703
704 uninit_q = blk_alloc_queue_node(GFP_KERNEL, node_id);
705 if (!uninit_q)
706 return NULL;
707
708 q = blk_init_allocated_queue(uninit_q, rfn, lock);
709 if (!q)
710 blk_cleanup_queue(uninit_q);
711
712 return q;
713}
714EXPORT_SYMBOL(blk_init_queue_node);
715
716struct request_queue *
717blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
718 spinlock_t *lock)
719{
720 if (!q)
721 return NULL;
722
723 if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
724 return NULL;
725
726 q->request_fn = rfn;
727 q->prep_rq_fn = NULL;
728 q->unprep_rq_fn = NULL;
729 q->queue_flags |= QUEUE_FLAG_DEFAULT;
730
731
732 if (lock)
733 q->queue_lock = lock;
734
735
736
737
738 blk_queue_make_request(q, blk_queue_bio);
739
740 q->sg_reserved_size = INT_MAX;
741
742
743 if (elevator_init(q, NULL))
744 return NULL;
745 return q;
746}
747EXPORT_SYMBOL(blk_init_allocated_queue);
748
749bool blk_get_queue(struct request_queue *q)
750{
751 if (likely(!blk_queue_dying(q))) {
752 __blk_get_queue(q);
753 return true;
754 }
755
756 return false;
757}
758EXPORT_SYMBOL(blk_get_queue);
759
760static inline void blk_free_request(struct request_list *rl, struct request *rq)
761{
762 if (rq->cmd_flags & REQ_ELVPRIV) {
763 elv_put_request(rl->q, rq);
764 if (rq->elv.icq)
765 put_io_context(rq->elv.icq->ioc);
766 }
767
768 mempool_free(rq, rl->rq_pool);
769}
770
771
772
773
774
775static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
776{
777 if (!ioc)
778 return 0;
779
780
781
782
783
784
785 return ioc->nr_batch_requests == q->nr_batching ||
786 (ioc->nr_batch_requests > 0
787 && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
788}
789
790
791
792
793
794
795
796static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
797{
798 if (!ioc || ioc_batching(q, ioc))
799 return;
800
801 ioc->nr_batch_requests = q->nr_batching;
802 ioc->last_waited = jiffies;
803}
804
805static void __freed_request(struct request_list *rl, int sync)
806{
807 struct request_queue *q = rl->q;
808
809
810
811
812
813 if (rl == &q->root_rl &&
814 rl->count[sync] < queue_congestion_off_threshold(q))
815 blk_clear_queue_congested(q, sync);
816
817 if (rl->count[sync] + 1 <= q->nr_requests) {
818 if (waitqueue_active(&rl->wait[sync]))
819 wake_up(&rl->wait[sync]);
820
821 blk_clear_rl_full(rl, sync);
822 }
823}
824
825
826
827
828
829static void freed_request(struct request_list *rl, unsigned int flags)
830{
831 struct request_queue *q = rl->q;
832 int sync = rw_is_sync(flags);
833
834 q->nr_rqs[sync]--;
835 rl->count[sync]--;
836 if (flags & REQ_ELVPRIV)
837 q->nr_rqs_elvpriv--;
838
839 __freed_request(rl, sync);
840
841 if (unlikely(rl->starved[sync ^ 1]))
842 __freed_request(rl, sync ^ 1);
843}
844
845
846
847
848
849static bool blk_rq_should_init_elevator(struct bio *bio)
850{
851 if (!bio)
852 return true;
853
854
855
856
857
858 if (bio->bi_rw & (REQ_FLUSH | REQ_FUA))
859 return false;
860
861 return true;
862}
863
864
865
866
867
868
869
870
871static struct io_context *rq_ioc(struct bio *bio)
872{
873#ifdef CONFIG_BLK_CGROUP
874 if (bio && bio->bi_ioc)
875 return bio->bi_ioc;
876#endif
877 return current->io_context;
878}
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894static struct request *__get_request(struct request_list *rl, int rw_flags,
895 struct bio *bio, gfp_t gfp_mask)
896{
897 struct request_queue *q = rl->q;
898 struct request *rq;
899 struct elevator_type *et = q->elevator->type;
900 struct io_context *ioc = rq_ioc(bio);
901 struct io_cq *icq = NULL;
902 const bool is_sync = rw_is_sync(rw_flags) != 0;
903 int may_queue;
904
905 if (unlikely(blk_queue_dying(q)))
906 return NULL;
907
908 may_queue = elv_may_queue(q, rw_flags);
909 if (may_queue == ELV_MQUEUE_NO)
910 goto rq_starved;
911
912 if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
913 if (rl->count[is_sync]+1 >= q->nr_requests) {
914
915
916
917
918
919
920 if (!blk_rl_full(rl, is_sync)) {
921 ioc_set_batching(q, ioc);
922 blk_set_rl_full(rl, is_sync);
923 } else {
924 if (may_queue != ELV_MQUEUE_MUST
925 && !ioc_batching(q, ioc)) {
926
927
928
929
930
931 return NULL;
932 }
933 }
934 }
935
936
937
938
939 if (rl == &q->root_rl)
940 blk_set_queue_congested(q, is_sync);
941 }
942
943
944
945
946
947
948 if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
949 return NULL;
950
951 q->nr_rqs[is_sync]++;
952 rl->count[is_sync]++;
953 rl->starved[is_sync] = 0;
954
955
956
957
958
959
960
961
962
963
964
965 if (blk_rq_should_init_elevator(bio) && !blk_queue_bypass(q)) {
966 rw_flags |= REQ_ELVPRIV;
967 q->nr_rqs_elvpriv++;
968 if (et->icq_cache && ioc)
969 icq = ioc_lookup_icq(ioc, q);
970 }
971
972 if (blk_queue_io_stat(q))
973 rw_flags |= REQ_IO_STAT;
974 spin_unlock_irq(q->queue_lock);
975
976
977 rq = mempool_alloc(rl->rq_pool, gfp_mask);
978 if (!rq)
979 goto fail_alloc;
980
981 blk_rq_init(q, rq);
982 blk_rq_set_rl(rq, rl);
983 rq->cmd_flags = rw_flags | REQ_ALLOCED;
984
985
986 if (rw_flags & REQ_ELVPRIV) {
987 if (unlikely(et->icq_cache && !icq)) {
988 if (ioc)
989 icq = ioc_create_icq(ioc, q, gfp_mask);
990 if (!icq)
991 goto fail_elvpriv;
992 }
993
994 rq->elv.icq = icq;
995 if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
996 goto fail_elvpriv;
997
998
999 if (icq)
1000 get_io_context(icq->ioc);
1001 }
1002out:
1003
1004
1005
1006
1007
1008
1009 if (ioc_batching(q, ioc))
1010 ioc->nr_batch_requests--;
1011
1012 trace_block_getrq(q, bio, rw_flags & 1);
1013 return rq;
1014
1015fail_elvpriv:
1016
1017
1018
1019
1020
1021
1022 printk_ratelimited(KERN_WARNING "%s: request aux data allocation failed, iosched may be disturbed\n",
1023 dev_name(q->backing_dev_info.dev));
1024
1025 rq->cmd_flags &= ~REQ_ELVPRIV;
1026 rq->elv.icq = NULL;
1027
1028 spin_lock_irq(q->queue_lock);
1029 q->nr_rqs_elvpriv--;
1030 spin_unlock_irq(q->queue_lock);
1031 goto out;
1032
1033fail_alloc:
1034
1035
1036
1037
1038
1039
1040
1041 spin_lock_irq(q->queue_lock);
1042 freed_request(rl, rw_flags);
1043
1044
1045
1046
1047
1048
1049
1050
1051rq_starved:
1052 if (unlikely(rl->count[is_sync] == 0))
1053 rl->starved[is_sync] = 1;
1054 return NULL;
1055}
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071static struct request *get_request(struct request_queue *q, int rw_flags,
1072 struct bio *bio, gfp_t gfp_mask)
1073{
1074 const bool is_sync = rw_is_sync(rw_flags) != 0;
1075 DEFINE_WAIT(wait);
1076 struct request_list *rl;
1077 struct request *rq;
1078
1079 rl = blk_get_rl(q, bio);
1080retry:
1081 rq = __get_request(rl, rw_flags, bio, gfp_mask);
1082 if (rq)
1083 return rq;
1084
1085 if (!(gfp_mask & __GFP_WAIT) || unlikely(blk_queue_dying(q))) {
1086 blk_put_rl(rl);
1087 return NULL;
1088 }
1089
1090
1091 prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
1092 TASK_UNINTERRUPTIBLE);
1093
1094 trace_block_sleeprq(q, bio, rw_flags & 1);
1095
1096 spin_unlock_irq(q->queue_lock);
1097 io_schedule();
1098
1099
1100
1101
1102
1103
1104 ioc_set_batching(q, current->io_context);
1105
1106 spin_lock_irq(q->queue_lock);
1107 finish_wait(&rl->wait[is_sync], &wait);
1108
1109 goto retry;
1110}
1111
1112struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
1113{
1114 struct request *rq;
1115
1116 BUG_ON(rw != READ && rw != WRITE);
1117
1118
1119 create_io_context(gfp_mask, q->node);
1120
1121 spin_lock_irq(q->queue_lock);
1122 rq = get_request(q, rw, NULL, gfp_mask);
1123 if (!rq)
1124 spin_unlock_irq(q->queue_lock);
1125
1126
1127 return rq;
1128}
1129EXPORT_SYMBOL(blk_get_request);
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162struct request *blk_make_request(struct request_queue *q, struct bio *bio,
1163 gfp_t gfp_mask)
1164{
1165 struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask);
1166
1167 if (unlikely(!rq))
1168 return ERR_PTR(-ENOMEM);
1169
1170 for_each_bio(bio) {
1171 struct bio *bounce_bio = bio;
1172 int ret;
1173
1174 blk_queue_bounce(q, &bounce_bio);
1175 ret = blk_rq_append_bio(q, rq, bounce_bio);
1176 if (unlikely(ret)) {
1177 blk_put_request(rq);
1178 return ERR_PTR(ret);
1179 }
1180 }
1181
1182 return rq;
1183}
1184EXPORT_SYMBOL(blk_make_request);
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196void blk_requeue_request(struct request_queue *q, struct request *rq)
1197{
1198 blk_delete_timer(rq);
1199 blk_clear_rq_complete(rq);
1200 trace_block_rq_requeue(q, rq);
1201
1202 if (blk_rq_tagged(rq))
1203 blk_queue_end_tag(q, rq);
1204
1205 BUG_ON(blk_queued_rq(rq));
1206
1207 elv_requeue_request(q, rq);
1208}
1209EXPORT_SYMBOL(blk_requeue_request);
1210
1211static void add_acct_request(struct request_queue *q, struct request *rq,
1212 int where)
1213{
1214 drive_stat_acct(rq, 1);
1215 __elv_add_request(q, rq, where);
1216}
1217
1218static void part_round_stats_single(int cpu, struct hd_struct *part,
1219 unsigned long now)
1220{
1221 if (now == part->stamp)
1222 return;
1223
1224 if (part_in_flight(part)) {
1225 __part_stat_add(cpu, part, time_in_queue,
1226 part_in_flight(part) * (now - part->stamp));
1227 __part_stat_add(cpu, part, io_ticks, (now - part->stamp));
1228 }
1229 part->stamp = now;
1230}
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248void part_round_stats(int cpu, struct hd_struct *part)
1249{
1250 unsigned long now = jiffies;
1251
1252 if (part->partno)
1253 part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
1254 part_round_stats_single(cpu, part, now);
1255}
1256EXPORT_SYMBOL_GPL(part_round_stats);
1257
1258#ifdef CONFIG_PM_RUNTIME
1259static void blk_pm_put_request(struct request *rq)
1260{
1261 if (rq->q->dev && !(rq->cmd_flags & REQ_PM) && !--rq->q->nr_pending)
1262 pm_runtime_mark_last_busy(rq->q->dev);
1263}
1264#else
1265static inline void blk_pm_put_request(struct request *rq) {}
1266#endif
1267
1268
1269
1270
1271void __blk_put_request(struct request_queue *q, struct request *req)
1272{
1273 if (unlikely(!q))
1274 return;
1275 if (unlikely(--req->ref_count))
1276 return;
1277
1278 blk_pm_put_request(req);
1279
1280 elv_completed_request(q, req);
1281
1282
1283 WARN_ON(req->bio != NULL);
1284
1285
1286
1287
1288
1289 if (req->cmd_flags & REQ_ALLOCED) {
1290 unsigned int flags = req->cmd_flags;
1291 struct request_list *rl = blk_rq_rl(req);
1292
1293 BUG_ON(!list_empty(&req->queuelist));
1294 BUG_ON(!hlist_unhashed(&req->hash));
1295
1296 blk_free_request(rl, req);
1297 freed_request(rl, flags);
1298 blk_put_rl(rl);
1299 }
1300}
1301EXPORT_SYMBOL_GPL(__blk_put_request);
1302
1303void blk_put_request(struct request *req)
1304{
1305 unsigned long flags;
1306 struct request_queue *q = req->q;
1307
1308 spin_lock_irqsave(q->queue_lock, flags);
1309 __blk_put_request(q, req);
1310 spin_unlock_irqrestore(q->queue_lock, flags);
1311}
1312EXPORT_SYMBOL(blk_put_request);
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327void blk_add_request_payload(struct request *rq, struct page *page,
1328 unsigned int len)
1329{
1330 struct bio *bio = rq->bio;
1331
1332 bio->bi_io_vec->bv_page = page;
1333 bio->bi_io_vec->bv_offset = 0;
1334 bio->bi_io_vec->bv_len = len;
1335
1336 bio->bi_size = len;
1337 bio->bi_vcnt = 1;
1338 bio->bi_phys_segments = 1;
1339
1340 rq->__data_len = rq->resid_len = len;
1341 rq->nr_phys_segments = 1;
1342 rq->buffer = bio_data(bio);
1343}
1344EXPORT_SYMBOL_GPL(blk_add_request_payload);
1345
1346static bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
1347 struct bio *bio)
1348{
1349 const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
1350
1351 if (!ll_back_merge_fn(q, req, bio))
1352 return false;
1353
1354 trace_block_bio_backmerge(q, req, bio);
1355
1356 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1357 blk_rq_set_mixed_merge(req);
1358
1359 req->biotail->bi_next = bio;
1360 req->biotail = bio;
1361 req->__data_len += bio->bi_size;
1362 req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1363
1364 drive_stat_acct(req, 0);
1365 return true;
1366}
1367
1368static bool bio_attempt_front_merge(struct request_queue *q,
1369 struct request *req, struct bio *bio)
1370{
1371 const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
1372
1373 if (!ll_front_merge_fn(q, req, bio))
1374 return false;
1375
1376 trace_block_bio_frontmerge(q, req, bio);
1377
1378 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1379 blk_rq_set_mixed_merge(req);
1380
1381 bio->bi_next = req->bio;
1382 req->bio = bio;
1383
1384
1385
1386
1387
1388
1389 req->buffer = bio_data(bio);
1390 req->__sector = bio->bi_sector;
1391 req->__data_len += bio->bi_size;
1392 req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1393
1394 drive_stat_acct(req, 0);
1395 return true;
1396}
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415static bool attempt_plug_merge(struct request_queue *q, struct bio *bio,
1416 unsigned int *request_count)
1417{
1418 struct blk_plug *plug;
1419 struct request *rq;
1420 bool ret = false;
1421
1422 plug = current->plug;
1423 if (!plug)
1424 goto out;
1425 *request_count = 0;
1426
1427 list_for_each_entry_reverse(rq, &plug->list, queuelist) {
1428 int el_ret;
1429
1430 if (rq->q == q)
1431 (*request_count)++;
1432
1433 if (rq->q != q || !blk_rq_merge_ok(rq, bio))
1434 continue;
1435
1436 el_ret = blk_try_merge(rq, bio);
1437 if (el_ret == ELEVATOR_BACK_MERGE) {
1438 ret = bio_attempt_back_merge(q, rq, bio);
1439 if (ret)
1440 break;
1441 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
1442 ret = bio_attempt_front_merge(q, rq, bio);
1443 if (ret)
1444 break;
1445 }
1446 }
1447out:
1448 return ret;
1449}
1450
1451void init_request_from_bio(struct request *req, struct bio *bio)
1452{
1453 req->cmd_type = REQ_TYPE_FS;
1454
1455 req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK;
1456 if (bio->bi_rw & REQ_RAHEAD)
1457 req->cmd_flags |= REQ_FAILFAST_MASK;
1458
1459 req->errors = 0;
1460 req->__sector = bio->bi_sector;
1461 req->ioprio = bio_prio(bio);
1462 blk_rq_bio_prep(req->q, req, bio);
1463}
1464
1465void blk_queue_bio(struct request_queue *q, struct bio *bio)
1466{
1467 const bool sync = !!(bio->bi_rw & REQ_SYNC);
1468 struct blk_plug *plug;
1469 int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT;
1470 struct request *req;
1471 unsigned int request_count = 0;
1472
1473
1474
1475
1476
1477
1478 blk_queue_bounce(q, &bio);
1479
1480 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1481 bio_endio(bio, -EIO);
1482 return;
1483 }
1484
1485 if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
1486 spin_lock_irq(q->queue_lock);
1487 where = ELEVATOR_INSERT_FLUSH;
1488 goto get_rq;
1489 }
1490
1491
1492
1493
1494
1495 if (attempt_plug_merge(q, bio, &request_count))
1496 return;
1497
1498 spin_lock_irq(q->queue_lock);
1499
1500 el_ret = elv_merge(q, &req, bio);
1501 if (el_ret == ELEVATOR_BACK_MERGE) {
1502 if (bio_attempt_back_merge(q, req, bio)) {
1503 elv_bio_merged(q, req, bio);
1504 if (!attempt_back_merge(q, req))
1505 elv_merged_request(q, req, el_ret);
1506 goto out_unlock;
1507 }
1508 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
1509 if (bio_attempt_front_merge(q, req, bio)) {
1510 elv_bio_merged(q, req, bio);
1511 if (!attempt_front_merge(q, req))
1512 elv_merged_request(q, req, el_ret);
1513 goto out_unlock;
1514 }
1515 }
1516
1517get_rq:
1518
1519
1520
1521
1522
1523 rw_flags = bio_data_dir(bio);
1524 if (sync)
1525 rw_flags |= REQ_SYNC;
1526
1527
1528
1529
1530
1531 req = get_request(q, rw_flags, bio, GFP_NOIO);
1532 if (unlikely(!req)) {
1533 bio_endio(bio, -ENODEV);
1534 goto out_unlock;
1535 }
1536
1537
1538
1539
1540
1541
1542
1543 init_request_from_bio(req, bio);
1544
1545 if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags))
1546 req->cpu = raw_smp_processor_id();
1547
1548 plug = current->plug;
1549 if (plug) {
1550
1551
1552
1553
1554 if (!request_count)
1555 trace_block_plug(q);
1556 else {
1557 if (request_count >= BLK_MAX_REQUEST_COUNT) {
1558 blk_flush_plug_list(plug, false);
1559 trace_block_plug(q);
1560 }
1561 }
1562 list_add_tail(&req->queuelist, &plug->list);
1563 drive_stat_acct(req, 1);
1564 } else {
1565 spin_lock_irq(q->queue_lock);
1566 add_acct_request(q, req, where);
1567 __blk_run_queue(q);
1568out_unlock:
1569 spin_unlock_irq(q->queue_lock);
1570 }
1571}
1572EXPORT_SYMBOL_GPL(blk_queue_bio);
1573
1574
1575
1576
1577static inline void blk_partition_remap(struct bio *bio)
1578{
1579 struct block_device *bdev = bio->bi_bdev;
1580
1581 if (bio_sectors(bio) && bdev != bdev->bd_contains) {
1582 struct hd_struct *p = bdev->bd_part;
1583
1584 bio->bi_sector += p->start_sect;
1585 bio->bi_bdev = bdev->bd_contains;
1586
1587 trace_block_bio_remap(bdev_get_queue(bio->bi_bdev), bio,
1588 bdev->bd_dev,
1589 bio->bi_sector - p->start_sect);
1590 }
1591}
1592
1593static void handle_bad_sector(struct bio *bio)
1594{
1595 char b[BDEVNAME_SIZE];
1596
1597 printk(KERN_INFO "attempt to access beyond end of device\n");
1598 printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n",
1599 bdevname(bio->bi_bdev, b),
1600 bio->bi_rw,
1601 (unsigned long long)bio_end_sector(bio),
1602 (long long)(i_size_read(bio->bi_bdev->bd_inode) >> 9));
1603
1604 set_bit(BIO_EOF, &bio->bi_flags);
1605}
1606
1607#ifdef CONFIG_FAIL_MAKE_REQUEST
1608
1609static DECLARE_FAULT_ATTR(fail_make_request);
1610
1611static int __init setup_fail_make_request(char *str)
1612{
1613 return setup_fault_attr(&fail_make_request, str);
1614}
1615__setup("fail_make_request=", setup_fail_make_request);
1616
1617static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
1618{
1619 return part->make_it_fail && should_fail(&fail_make_request, bytes);
1620}
1621
1622static int __init fail_make_request_debugfs(void)
1623{
1624 struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
1625 NULL, &fail_make_request);
1626
1627 return IS_ERR(dir) ? PTR_ERR(dir) : 0;
1628}
1629
1630late_initcall(fail_make_request_debugfs);
1631
1632#else
1633
1634static inline bool should_fail_request(struct hd_struct *part,
1635 unsigned int bytes)
1636{
1637 return false;
1638}
1639
1640#endif
1641
1642
1643
1644
1645static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
1646{
1647 sector_t maxsector;
1648
1649 if (!nr_sectors)
1650 return 0;
1651
1652
1653 maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9;
1654 if (maxsector) {
1655 sector_t sector = bio->bi_sector;
1656
1657 if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
1658
1659
1660
1661
1662
1663 handle_bad_sector(bio);
1664 return 1;
1665 }
1666 }
1667
1668 return 0;
1669}
1670
1671static noinline_for_stack bool
1672generic_make_request_checks(struct bio *bio)
1673{
1674 struct request_queue *q;
1675 int nr_sectors = bio_sectors(bio);
1676 int err = -EIO;
1677 char b[BDEVNAME_SIZE];
1678 struct hd_struct *part;
1679
1680 might_sleep();
1681
1682 if (bio_check_eod(bio, nr_sectors))
1683 goto end_io;
1684
1685 q = bdev_get_queue(bio->bi_bdev);
1686 if (unlikely(!q)) {
1687 printk(KERN_ERR
1688 "generic_make_request: Trying to access "
1689 "nonexistent block-device %s (%Lu)\n",
1690 bdevname(bio->bi_bdev, b),
1691 (long long) bio->bi_sector);
1692 goto end_io;
1693 }
1694
1695 if (likely(bio_is_rw(bio) &&
1696 nr_sectors > queue_max_hw_sectors(q))) {
1697 printk(KERN_ERR "bio too big device %s (%u > %u)\n",
1698 bdevname(bio->bi_bdev, b),
1699 bio_sectors(bio),
1700 queue_max_hw_sectors(q));
1701 goto end_io;
1702 }
1703
1704 part = bio->bi_bdev->bd_part;
1705 if (should_fail_request(part, bio->bi_size) ||
1706 should_fail_request(&part_to_disk(part)->part0,
1707 bio->bi_size))
1708 goto end_io;
1709
1710
1711
1712
1713
1714 blk_partition_remap(bio);
1715
1716 if (bio_check_eod(bio, nr_sectors))
1717 goto end_io;
1718
1719
1720
1721
1722
1723
1724 if ((bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && !q->flush_flags) {
1725 bio->bi_rw &= ~(REQ_FLUSH | REQ_FUA);
1726 if (!nr_sectors) {
1727 err = 0;
1728 goto end_io;
1729 }
1730 }
1731
1732 if ((bio->bi_rw & REQ_DISCARD) &&
1733 (!blk_queue_discard(q) ||
1734 ((bio->bi_rw & REQ_SECURE) && !blk_queue_secdiscard(q)))) {
1735 err = -EOPNOTSUPP;
1736 goto end_io;
1737 }
1738
1739 if (bio->bi_rw & REQ_WRITE_SAME && !bdev_write_same(bio->bi_bdev)) {
1740 err = -EOPNOTSUPP;
1741 goto end_io;
1742 }
1743
1744
1745
1746
1747
1748
1749
1750 create_io_context(GFP_ATOMIC, q->node);
1751
1752 if (blk_throtl_bio(q, bio))
1753 return false;
1754
1755 trace_block_bio_queue(q, bio);
1756 return true;
1757
1758end_io:
1759 bio_endio(bio, err);
1760 return false;
1761}
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787void generic_make_request(struct bio *bio)
1788{
1789 struct bio_list bio_list_on_stack;
1790
1791 if (!generic_make_request_checks(bio))
1792 return;
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804 if (current->bio_list) {
1805 bio_list_add(current->bio_list, bio);
1806 return;
1807 }
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823 BUG_ON(bio->bi_next);
1824 bio_list_init(&bio_list_on_stack);
1825 current->bio_list = &bio_list_on_stack;
1826 do {
1827 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
1828
1829 q->make_request_fn(q, bio);
1830
1831 bio = bio_list_pop(current->bio_list);
1832 } while (bio);
1833 current->bio_list = NULL;
1834}
1835EXPORT_SYMBOL(generic_make_request);
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847void submit_bio(int rw, struct bio *bio)
1848{
1849 bio->bi_rw |= rw;
1850
1851
1852
1853
1854
1855 if (bio_has_data(bio)) {
1856 unsigned int count;
1857
1858 if (unlikely(rw & REQ_WRITE_SAME))
1859 count = bdev_logical_block_size(bio->bi_bdev) >> 9;
1860 else
1861 count = bio_sectors(bio);
1862
1863 if (rw & WRITE) {
1864 count_vm_events(PGPGOUT, count);
1865 } else {
1866 task_io_account_read(bio->bi_size);
1867 count_vm_events(PGPGIN, count);
1868 }
1869
1870 if (unlikely(block_dump)) {
1871 char b[BDEVNAME_SIZE];
1872 printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
1873 current->comm, task_pid_nr(current),
1874 (rw & WRITE) ? "WRITE" : "READ",
1875 (unsigned long long)bio->bi_sector,
1876 bdevname(bio->bi_bdev, b),
1877 count);
1878 }
1879 }
1880
1881 generic_make_request(bio);
1882}
1883EXPORT_SYMBOL(submit_bio);
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906int blk_rq_check_limits(struct request_queue *q, struct request *rq)
1907{
1908 if (!rq_mergeable(rq))
1909 return 0;
1910
1911 if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, rq->cmd_flags)) {
1912 printk(KERN_ERR "%s: over max size limit.\n", __func__);
1913 return -EIO;
1914 }
1915
1916
1917
1918
1919
1920
1921
1922 blk_recalc_rq_segments(rq);
1923 if (rq->nr_phys_segments > queue_max_segments(q)) {
1924 printk(KERN_ERR "%s: over max segments limit.\n", __func__);
1925 return -EIO;
1926 }
1927
1928 return 0;
1929}
1930EXPORT_SYMBOL_GPL(blk_rq_check_limits);
1931
1932
1933
1934
1935
1936
1937int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
1938{
1939 unsigned long flags;
1940 int where = ELEVATOR_INSERT_BACK;
1941
1942 if (blk_rq_check_limits(q, rq))
1943 return -EIO;
1944
1945 if (rq->rq_disk &&
1946 should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
1947 return -EIO;
1948
1949 spin_lock_irqsave(q->queue_lock, flags);
1950 if (unlikely(blk_queue_dying(q))) {
1951 spin_unlock_irqrestore(q->queue_lock, flags);
1952 return -ENODEV;
1953 }
1954
1955
1956
1957
1958
1959 BUG_ON(blk_queued_rq(rq));
1960
1961 if (rq->cmd_flags & (REQ_FLUSH|REQ_FUA))
1962 where = ELEVATOR_INSERT_FLUSH;
1963
1964 add_acct_request(q, rq, where);
1965 if (where == ELEVATOR_INSERT_FLUSH)
1966 __blk_run_queue(q);
1967 spin_unlock_irqrestore(q->queue_lock, flags);
1968
1969 return 0;
1970}
1971EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989unsigned int blk_rq_err_bytes(const struct request *rq)
1990{
1991 unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
1992 unsigned int bytes = 0;
1993 struct bio *bio;
1994
1995 if (!(rq->cmd_flags & REQ_MIXED_MERGE))
1996 return blk_rq_bytes(rq);
1997
1998
1999
2000
2001
2002
2003
2004
2005 for (bio = rq->bio; bio; bio = bio->bi_next) {
2006 if ((bio->bi_rw & ff) != ff)
2007 break;
2008 bytes += bio->bi_size;
2009 }
2010
2011
2012 BUG_ON(blk_rq_bytes(rq) && !bytes);
2013 return bytes;
2014}
2015EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
2016
2017static void blk_account_io_completion(struct request *req, unsigned int bytes)
2018{
2019 if (blk_do_io_stat(req)) {
2020 const int rw = rq_data_dir(req);
2021 struct hd_struct *part;
2022 int cpu;
2023
2024 cpu = part_stat_lock();
2025 part = req->part;
2026 part_stat_add(cpu, part, sectors[rw], bytes >> 9);
2027 part_stat_unlock();
2028 }
2029}
2030
2031static void blk_account_io_done(struct request *req)
2032{
2033
2034
2035
2036
2037
2038 if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) {
2039 unsigned long duration = jiffies - req->start_time;
2040 const int rw = rq_data_dir(req);
2041 struct hd_struct *part;
2042 int cpu;
2043
2044 cpu = part_stat_lock();
2045 part = req->part;
2046
2047 part_stat_inc(cpu, part, ios[rw]);
2048 part_stat_add(cpu, part, ticks[rw], duration);
2049 part_round_stats(cpu, part);
2050 part_dec_in_flight(part, rw);
2051
2052 hd_struct_put(part);
2053 part_stat_unlock();
2054 }
2055}
2056
2057#ifdef CONFIG_PM_RUNTIME
2058
2059
2060
2061
2062static struct request *blk_pm_peek_request(struct request_queue *q,
2063 struct request *rq)
2064{
2065 if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
2066 (q->rpm_status != RPM_ACTIVE && !(rq->cmd_flags & REQ_PM))))
2067 return NULL;
2068 else
2069 return rq;
2070}
2071#else
2072static inline struct request *blk_pm_peek_request(struct request_queue *q,
2073 struct request *rq)
2074{
2075 return rq;
2076}
2077#endif
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095struct request *blk_peek_request(struct request_queue *q)
2096{
2097 struct request *rq;
2098 int ret;
2099
2100 while ((rq = __elv_next_request(q)) != NULL) {
2101
2102 rq = blk_pm_peek_request(q, rq);
2103 if (!rq)
2104 break;
2105
2106 if (!(rq->cmd_flags & REQ_STARTED)) {
2107
2108
2109
2110
2111
2112 if (rq->cmd_flags & REQ_SORTED)
2113 elv_activate_rq(q, rq);
2114
2115
2116
2117
2118
2119
2120 rq->cmd_flags |= REQ_STARTED;
2121 trace_block_rq_issue(q, rq);
2122 }
2123
2124 if (!q->boundary_rq || q->boundary_rq == rq) {
2125 q->end_sector = rq_end_sector(rq);
2126 q->boundary_rq = NULL;
2127 }
2128
2129 if (rq->cmd_flags & REQ_DONTPREP)
2130 break;
2131
2132 if (q->dma_drain_size && blk_rq_bytes(rq)) {
2133
2134
2135
2136
2137
2138
2139 rq->nr_phys_segments++;
2140 }
2141
2142 if (!q->prep_rq_fn)
2143 break;
2144
2145 ret = q->prep_rq_fn(q, rq);
2146 if (ret == BLKPREP_OK) {
2147 break;
2148 } else if (ret == BLKPREP_DEFER) {
2149
2150
2151
2152
2153
2154
2155 if (q->dma_drain_size && blk_rq_bytes(rq) &&
2156 !(rq->cmd_flags & REQ_DONTPREP)) {
2157
2158
2159
2160
2161 --rq->nr_phys_segments;
2162 }
2163
2164 rq = NULL;
2165 break;
2166 } else if (ret == BLKPREP_KILL) {
2167 rq->cmd_flags |= REQ_QUIET;
2168
2169
2170
2171
2172 blk_start_request(rq);
2173 __blk_end_request_all(rq, -EIO);
2174 } else {
2175 printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
2176 break;
2177 }
2178 }
2179
2180 return rq;
2181}
2182EXPORT_SYMBOL(blk_peek_request);
2183
2184void blk_dequeue_request(struct request *rq)
2185{
2186 struct request_queue *q = rq->q;
2187
2188 BUG_ON(list_empty(&rq->queuelist));
2189 BUG_ON(ELV_ON_HASH(rq));
2190
2191 list_del_init(&rq->queuelist);
2192
2193
2194
2195
2196
2197
2198 if (blk_account_rq(rq)) {
2199 q->in_flight[rq_is_sync(rq)]++;
2200 set_io_start_time_ns(rq);
2201 }
2202}
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218void blk_start_request(struct request *req)
2219{
2220 blk_dequeue_request(req);
2221
2222
2223
2224
2225
2226 req->resid_len = blk_rq_bytes(req);
2227 if (unlikely(blk_bidi_rq(req)))
2228 req->next_rq->resid_len = blk_rq_bytes(req->next_rq);
2229
2230 blk_add_timer(req);
2231}
2232EXPORT_SYMBOL(blk_start_request);
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249struct request *blk_fetch_request(struct request_queue *q)
2250{
2251 struct request *rq;
2252
2253 rq = blk_peek_request(q);
2254 if (rq)
2255 blk_start_request(rq);
2256 return rq;
2257}
2258EXPORT_SYMBOL(blk_fetch_request);
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
2283{
2284 int total_bytes;
2285
2286 if (!req->bio)
2287 return false;
2288
2289 trace_block_rq_complete(req->q, req);
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299 if (req->cmd_type == REQ_TYPE_FS)
2300 req->errors = 0;
2301
2302 if (error && req->cmd_type == REQ_TYPE_FS &&
2303 !(req->cmd_flags & REQ_QUIET)) {
2304 char *error_type;
2305
2306 switch (error) {
2307 case -ENOLINK:
2308 error_type = "recoverable transport";
2309 break;
2310 case -EREMOTEIO:
2311 error_type = "critical target";
2312 break;
2313 case -EBADE:
2314 error_type = "critical nexus";
2315 break;
2316 case -ETIMEDOUT:
2317 error_type = "timeout";
2318 break;
2319 case -ENOSPC:
2320 error_type = "critical space allocation";
2321 break;
2322 case -ENODATA:
2323 error_type = "critical medium";
2324 break;
2325 case -EIO:
2326 default:
2327 error_type = "I/O";
2328 break;
2329 }
2330 printk_ratelimited(KERN_ERR "end_request: %s error, dev %s, sector %llu\n",
2331 error_type, req->rq_disk ?
2332 req->rq_disk->disk_name : "?",
2333 (unsigned long long)blk_rq_pos(req));
2334
2335 }
2336
2337 blk_account_io_completion(req, nr_bytes);
2338
2339 total_bytes = 0;
2340 while (req->bio) {
2341 struct bio *bio = req->bio;
2342 unsigned bio_bytes = min(bio->bi_size, nr_bytes);
2343
2344 if (bio_bytes == bio->bi_size)
2345 req->bio = bio->bi_next;
2346
2347 req_bio_endio(req, bio, bio_bytes, error);
2348
2349 total_bytes += bio_bytes;
2350 nr_bytes -= bio_bytes;
2351
2352 if (!nr_bytes)
2353 break;
2354 }
2355
2356
2357
2358
2359 if (!req->bio) {
2360
2361
2362
2363
2364
2365 req->__data_len = 0;
2366 return false;
2367 }
2368
2369 req->__data_len -= total_bytes;
2370 req->buffer = bio_data(req->bio);
2371
2372
2373 if (req->cmd_type == REQ_TYPE_FS)
2374 req->__sector += total_bytes >> 9;
2375
2376
2377 if (req->cmd_flags & REQ_MIXED_MERGE) {
2378 req->cmd_flags &= ~REQ_FAILFAST_MASK;
2379 req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK;
2380 }
2381
2382
2383
2384
2385
2386 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
2387 blk_dump_rq_flags(req, "request botched");
2388 req->__data_len = blk_rq_cur_bytes(req);
2389 }
2390
2391
2392 blk_recalc_rq_segments(req);
2393
2394 return true;
2395}
2396EXPORT_SYMBOL_GPL(blk_update_request);
2397
2398static bool blk_update_bidi_request(struct request *rq, int error,
2399 unsigned int nr_bytes,
2400 unsigned int bidi_bytes)
2401{
2402 if (blk_update_request(rq, error, nr_bytes))
2403 return true;
2404
2405
2406 if (unlikely(blk_bidi_rq(rq)) &&
2407 blk_update_request(rq->next_rq, error, bidi_bytes))
2408 return true;
2409
2410 if (blk_queue_add_random(rq->q))
2411 add_disk_randomness(rq->rq_disk);
2412
2413 return false;
2414}
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426void blk_unprep_request(struct request *req)
2427{
2428 struct request_queue *q = req->q;
2429
2430 req->cmd_flags &= ~REQ_DONTPREP;
2431 if (q->unprep_rq_fn)
2432 q->unprep_rq_fn(q, req);
2433}
2434EXPORT_SYMBOL_GPL(blk_unprep_request);
2435
2436
2437
2438
2439static void blk_finish_request(struct request *req, int error)
2440{
2441 if (blk_rq_tagged(req))
2442 blk_queue_end_tag(req->q, req);
2443
2444 BUG_ON(blk_queued_rq(req));
2445
2446 if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS)
2447 laptop_io_completion(&req->q->backing_dev_info);
2448
2449 blk_delete_timer(req);
2450
2451 if (req->cmd_flags & REQ_DONTPREP)
2452 blk_unprep_request(req);
2453
2454
2455 blk_account_io_done(req);
2456
2457 if (req->end_io)
2458 req->end_io(req, error);
2459 else {
2460 if (blk_bidi_rq(req))
2461 __blk_put_request(req->next_rq->q, req->next_rq);
2462
2463 __blk_put_request(req->q, req);
2464 }
2465}
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484static bool blk_end_bidi_request(struct request *rq, int error,
2485 unsigned int nr_bytes, unsigned int bidi_bytes)
2486{
2487 struct request_queue *q = rq->q;
2488 unsigned long flags;
2489
2490 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2491 return true;
2492
2493 spin_lock_irqsave(q->queue_lock, flags);
2494 blk_finish_request(rq, error);
2495 spin_unlock_irqrestore(q->queue_lock, flags);
2496
2497 return false;
2498}
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515bool __blk_end_bidi_request(struct request *rq, int error,
2516 unsigned int nr_bytes, unsigned int bidi_bytes)
2517{
2518 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2519 return true;
2520
2521 blk_finish_request(rq, error);
2522
2523 return false;
2524}
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2541{
2542 return blk_end_bidi_request(rq, error, nr_bytes, 0);
2543}
2544EXPORT_SYMBOL(blk_end_request);
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554void blk_end_request_all(struct request *rq, int error)
2555{
2556 bool pending;
2557 unsigned int bidi_bytes = 0;
2558
2559 if (unlikely(blk_bidi_rq(rq)))
2560 bidi_bytes = blk_rq_bytes(rq->next_rq);
2561
2562 pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2563 BUG_ON(pending);
2564}
2565EXPORT_SYMBOL(blk_end_request_all);
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579bool blk_end_request_cur(struct request *rq, int error)
2580{
2581 return blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2582}
2583EXPORT_SYMBOL(blk_end_request_cur);
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597bool blk_end_request_err(struct request *rq, int error)
2598{
2599 WARN_ON(error >= 0);
2600 return blk_end_request(rq, error, blk_rq_err_bytes(rq));
2601}
2602EXPORT_SYMBOL_GPL(blk_end_request_err);
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2618{
2619 return __blk_end_bidi_request(rq, error, nr_bytes, 0);
2620}
2621EXPORT_SYMBOL(__blk_end_request);
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631void __blk_end_request_all(struct request *rq, int error)
2632{
2633 bool pending;
2634 unsigned int bidi_bytes = 0;
2635
2636 if (unlikely(blk_bidi_rq(rq)))
2637 bidi_bytes = blk_rq_bytes(rq->next_rq);
2638
2639 pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2640 BUG_ON(pending);
2641}
2642EXPORT_SYMBOL(__blk_end_request_all);
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657bool __blk_end_request_cur(struct request *rq, int error)
2658{
2659 return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2660}
2661EXPORT_SYMBOL(__blk_end_request_cur);
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676bool __blk_end_request_err(struct request *rq, int error)
2677{
2678 WARN_ON(error >= 0);
2679 return __blk_end_request(rq, error, blk_rq_err_bytes(rq));
2680}
2681EXPORT_SYMBOL_GPL(__blk_end_request_err);
2682
2683void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
2684 struct bio *bio)
2685{
2686
2687 rq->cmd_flags |= bio->bi_rw & REQ_WRITE;
2688
2689 if (bio_has_data(bio)) {
2690 rq->nr_phys_segments = bio_phys_segments(q, bio);
2691 rq->buffer = bio_data(bio);
2692 }
2693 rq->__data_len = bio->bi_size;
2694 rq->bio = rq->biotail = bio;
2695
2696 if (bio->bi_bdev)
2697 rq->rq_disk = bio->bi_bdev->bd_disk;
2698}
2699
2700#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
2701
2702
2703
2704
2705
2706
2707
2708void rq_flush_dcache_pages(struct request *rq)
2709{
2710 struct req_iterator iter;
2711 struct bio_vec *bvec;
2712
2713 rq_for_each_segment(bvec, rq, iter)
2714 flush_dcache_page(bvec->bv_page);
2715}
2716EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
2717#endif
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738int blk_lld_busy(struct request_queue *q)
2739{
2740 if (q->lld_busy_fn)
2741 return q->lld_busy_fn(q);
2742
2743 return 0;
2744}
2745EXPORT_SYMBOL_GPL(blk_lld_busy);
2746
2747
2748
2749
2750
2751
2752
2753
2754void blk_rq_unprep_clone(struct request *rq)
2755{
2756 struct bio *bio;
2757
2758 while ((bio = rq->bio) != NULL) {
2759 rq->bio = bio->bi_next;
2760
2761 bio_put(bio);
2762 }
2763}
2764EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
2765
2766
2767
2768
2769
2770static void __blk_rq_prep_clone(struct request *dst, struct request *src)
2771{
2772 dst->cpu = src->cpu;
2773 dst->cmd_flags = (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE;
2774 dst->cmd_type = src->cmd_type;
2775 dst->__sector = blk_rq_pos(src);
2776 dst->__data_len = blk_rq_bytes(src);
2777 dst->nr_phys_segments = src->nr_phys_segments;
2778 dst->ioprio = src->ioprio;
2779 dst->extra_len = src->extra_len;
2780}
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
2802 struct bio_set *bs, gfp_t gfp_mask,
2803 int (*bio_ctr)(struct bio *, struct bio *, void *),
2804 void *data)
2805{
2806 struct bio *bio, *bio_src;
2807
2808 if (!bs)
2809 bs = fs_bio_set;
2810
2811 blk_rq_init(NULL, rq);
2812
2813 __rq_for_each_bio(bio_src, rq_src) {
2814 bio = bio_clone_bioset(bio_src, gfp_mask, bs);
2815 if (!bio)
2816 goto free_and_out;
2817
2818 if (bio_ctr && bio_ctr(bio, bio_src, data))
2819 goto free_and_out;
2820
2821 if (rq->bio) {
2822 rq->biotail->bi_next = bio;
2823 rq->biotail = bio;
2824 } else
2825 rq->bio = rq->biotail = bio;
2826 }
2827
2828 __blk_rq_prep_clone(rq, rq_src);
2829
2830 return 0;
2831
2832free_and_out:
2833 if (bio)
2834 bio_put(bio);
2835 blk_rq_unprep_clone(rq);
2836
2837 return -ENOMEM;
2838}
2839EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
2840
2841int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
2842{
2843 return queue_work(kblockd_workqueue, work);
2844}
2845EXPORT_SYMBOL(kblockd_schedule_work);
2846
2847int kblockd_schedule_delayed_work(struct request_queue *q,
2848 struct delayed_work *dwork, unsigned long delay)
2849{
2850 return queue_delayed_work(kblockd_workqueue, dwork, delay);
2851}
2852EXPORT_SYMBOL(kblockd_schedule_delayed_work);
2853
2854#define PLUG_MAGIC 0x91827364
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870void blk_start_plug(struct blk_plug *plug)
2871{
2872 struct task_struct *tsk = current;
2873
2874 plug->magic = PLUG_MAGIC;
2875 INIT_LIST_HEAD(&plug->list);
2876 INIT_LIST_HEAD(&plug->cb_list);
2877
2878
2879
2880
2881
2882 if (!tsk->plug) {
2883
2884
2885
2886
2887 tsk->plug = plug;
2888 }
2889}
2890EXPORT_SYMBOL(blk_start_plug);
2891
2892static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
2893{
2894 struct request *rqa = container_of(a, struct request, queuelist);
2895 struct request *rqb = container_of(b, struct request, queuelist);
2896
2897 return !(rqa->q < rqb->q ||
2898 (rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb)));
2899}
2900
2901
2902
2903
2904
2905
2906
2907static void queue_unplugged(struct request_queue *q, unsigned int depth,
2908 bool from_schedule)
2909 __releases(q->queue_lock)
2910{
2911 trace_block_unplug(q, depth, !from_schedule);
2912
2913 if (from_schedule)
2914 blk_run_queue_async(q);
2915 else
2916 __blk_run_queue(q);
2917 spin_unlock(q->queue_lock);
2918}
2919
2920static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
2921{
2922 LIST_HEAD(callbacks);
2923
2924 while (!list_empty(&plug->cb_list)) {
2925 list_splice_init(&plug->cb_list, &callbacks);
2926
2927 while (!list_empty(&callbacks)) {
2928 struct blk_plug_cb *cb = list_first_entry(&callbacks,
2929 struct blk_plug_cb,
2930 list);
2931 list_del(&cb->list);
2932 cb->callback(cb, from_schedule);
2933 }
2934 }
2935}
2936
2937struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
2938 int size)
2939{
2940 struct blk_plug *plug = current->plug;
2941 struct blk_plug_cb *cb;
2942
2943 if (!plug)
2944 return NULL;
2945
2946 list_for_each_entry(cb, &plug->cb_list, list)
2947 if (cb->callback == unplug && cb->data == data)
2948 return cb;
2949
2950
2951 BUG_ON(size < sizeof(*cb));
2952 cb = kzalloc(size, GFP_ATOMIC);
2953 if (cb) {
2954 cb->data = data;
2955 cb->callback = unplug;
2956 list_add(&cb->list, &plug->cb_list);
2957 }
2958 return cb;
2959}
2960EXPORT_SYMBOL(blk_check_plugged);
2961
2962void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
2963{
2964 struct request_queue *q;
2965 unsigned long flags;
2966 struct request *rq;
2967 LIST_HEAD(list);
2968 unsigned int depth;
2969
2970 BUG_ON(plug->magic != PLUG_MAGIC);
2971
2972 flush_plug_callbacks(plug, from_schedule);
2973 if (list_empty(&plug->list))
2974 return;
2975
2976 list_splice_init(&plug->list, &list);
2977
2978 list_sort(NULL, &list, plug_rq_cmp);
2979
2980 q = NULL;
2981 depth = 0;
2982
2983
2984
2985
2986
2987 local_irq_save(flags);
2988 while (!list_empty(&list)) {
2989 rq = list_entry_rq(list.next);
2990 list_del_init(&rq->queuelist);
2991 BUG_ON(!rq->q);
2992 if (rq->q != q) {
2993
2994
2995
2996 if (q)
2997 queue_unplugged(q, depth, from_schedule);
2998 q = rq->q;
2999 depth = 0;
3000 spin_lock(q->queue_lock);
3001 }
3002
3003
3004
3005
3006 if (unlikely(blk_queue_dying(q))) {
3007 __blk_end_request_all(rq, -ENODEV);
3008 continue;
3009 }
3010
3011
3012
3013
3014 if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA))
3015 __elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH);
3016 else
3017 __elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE);
3018
3019 depth++;
3020 }
3021
3022
3023
3024
3025 if (q)
3026 queue_unplugged(q, depth, from_schedule);
3027
3028 local_irq_restore(flags);
3029}
3030
3031void blk_finish_plug(struct blk_plug *plug)
3032{
3033 blk_flush_plug_list(plug, false);
3034
3035 if (plug == current->plug)
3036 current->plug = NULL;
3037}
3038EXPORT_SYMBOL(blk_finish_plug);
3039
3040#ifdef CONFIG_PM_RUNTIME
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062void blk_pm_runtime_init(struct request_queue *q, struct device *dev)
3063{
3064 q->dev = dev;
3065 q->rpm_status = RPM_ACTIVE;
3066 pm_runtime_set_autosuspend_delay(q->dev, -1);
3067 pm_runtime_use_autosuspend(q->dev);
3068}
3069EXPORT_SYMBOL(blk_pm_runtime_init);
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092int blk_pre_runtime_suspend(struct request_queue *q)
3093{
3094 int ret = 0;
3095
3096 spin_lock_irq(q->queue_lock);
3097 if (q->nr_pending) {
3098 ret = -EBUSY;
3099 pm_runtime_mark_last_busy(q->dev);
3100 } else {
3101 q->rpm_status = RPM_SUSPENDING;
3102 }
3103 spin_unlock_irq(q->queue_lock);
3104 return ret;
3105}
3106EXPORT_SYMBOL(blk_pre_runtime_suspend);
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121void blk_post_runtime_suspend(struct request_queue *q, int err)
3122{
3123 spin_lock_irq(q->queue_lock);
3124 if (!err) {
3125 q->rpm_status = RPM_SUSPENDED;
3126 } else {
3127 q->rpm_status = RPM_ACTIVE;
3128 pm_runtime_mark_last_busy(q->dev);
3129 }
3130 spin_unlock_irq(q->queue_lock);
3131}
3132EXPORT_SYMBOL(blk_post_runtime_suspend);
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145void blk_pre_runtime_resume(struct request_queue *q)
3146{
3147 spin_lock_irq(q->queue_lock);
3148 q->rpm_status = RPM_RESUMING;
3149 spin_unlock_irq(q->queue_lock);
3150}
3151EXPORT_SYMBOL(blk_pre_runtime_resume);
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167void blk_post_runtime_resume(struct request_queue *q, int err)
3168{
3169 spin_lock_irq(q->queue_lock);
3170 if (!err) {
3171 q->rpm_status = RPM_ACTIVE;
3172 __blk_run_queue(q);
3173 pm_runtime_mark_last_busy(q->dev);
3174 pm_request_autosuspend(q->dev);
3175 } else {
3176 q->rpm_status = RPM_SUSPENDED;
3177 }
3178 spin_unlock_irq(q->queue_lock);
3179}
3180EXPORT_SYMBOL(blk_post_runtime_resume);
3181#endif
3182
3183int __init blk_dev_init(void)
3184{
3185 BUILD_BUG_ON(__REQ_NR_BITS > 8 *
3186 sizeof(((struct request *)0)->cmd_flags));
3187
3188
3189 kblockd_workqueue = alloc_workqueue("kblockd",
3190 WQ_MEM_RECLAIM | WQ_HIGHPRI |
3191 WQ_POWER_EFFICIENT, 0);
3192 if (!kblockd_workqueue)
3193 panic("Failed to create kblockd\n");
3194
3195 request_cachep = kmem_cache_create("blkdev_requests",
3196 sizeof(struct request), 0, SLAB_PANIC, NULL);
3197
3198 blk_requestq_cachep = kmem_cache_create("blkdev_queue",
3199 sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
3200
3201 return 0;
3202}
3203