1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38#include <linux/interrupt.h>
39#include <linux/blkdev.h>
40#include <linux/blk-mq.h>
41#include <linux/hdreg.h>
42#include <linux/cdrom.h>
43#include <linux/module.h>
44#include <linux/slab.h>
45#include <linux/mutex.h>
46#include <linux/scatterlist.h>
47#include <linux/bitmap.h>
48#include <linux/list.h>
49
50#include <xen/xen.h>
51#include <xen/xenbus.h>
52#include <xen/grant_table.h>
53#include <xen/events.h>
54#include <xen/page.h>
55#include <xen/platform_pci.h>
56
57#include <xen/interface/grant_table.h>
58#include <xen/interface/io/blkif.h>
59#include <xen/interface/io/protocols.h>
60
61#include <asm/xen/hypervisor.h>
62
63
64
65
66
67
68
69
70
71
72
73
74
75#define HAS_EXTRA_REQ (BLKIF_MAX_SEGMENTS_PER_REQUEST < XEN_PFN_PER_PAGE)
76
77enum blkif_state {
78 BLKIF_STATE_DISCONNECTED,
79 BLKIF_STATE_CONNECTED,
80 BLKIF_STATE_SUSPENDED,
81};
82
83struct grant {
84 grant_ref_t gref;
85 struct page *page;
86 struct list_head node;
87};
88
89enum blk_req_status {
90 REQ_WAITING,
91 REQ_DONE,
92 REQ_ERROR,
93 REQ_EOPNOTSUPP,
94};
95
96struct blk_shadow {
97 struct blkif_request req;
98 struct request *request;
99 struct grant **grants_used;
100 struct grant **indirect_grants;
101 struct scatterlist *sg;
102 unsigned int num_sg;
103 enum blk_req_status status;
104
105 #define NO_ASSOCIATED_ID ~0UL
106
107
108
109
110 unsigned long associated_id;
111};
112
113struct blkif_req {
114 blk_status_t error;
115};
116
117static inline struct blkif_req *blkif_req(struct request *rq)
118{
119 return blk_mq_rq_to_pdu(rq);
120}
121
122static DEFINE_MUTEX(blkfront_mutex);
123static const struct block_device_operations xlvbd_block_fops;
124
125
126
127
128
129
130
131static unsigned int xen_blkif_max_segments = 32;
132module_param_named(max_indirect_segments, xen_blkif_max_segments, uint,
133 S_IRUGO);
134MODULE_PARM_DESC(max_indirect_segments,
135 "Maximum amount of segments in indirect requests (default is 32)");
136
137static unsigned int xen_blkif_max_queues = 4;
138module_param_named(max_queues, xen_blkif_max_queues, uint, S_IRUGO);
139MODULE_PARM_DESC(max_queues, "Maximum number of hardware queues/rings used per virtual disk");
140
141
142
143
144
145static unsigned int xen_blkif_max_ring_order;
146module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, S_IRUGO);
147MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring");
148
149#define BLK_RING_SIZE(info) \
150 __CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * (info)->nr_ring_pages)
151
152#define BLK_MAX_RING_SIZE \
153 __CONST_RING_SIZE(blkif, XEN_PAGE_SIZE * XENBUS_MAX_RING_GRANTS)
154
155
156
157
158
159#define RINGREF_NAME_LEN (20)
160
161
162
163#define QUEUE_NAME_LEN (17)
164
165
166
167
168
169
170struct blkfront_ring_info {
171
172 spinlock_t ring_lock;
173 struct blkif_front_ring ring;
174 unsigned int ring_ref[XENBUS_MAX_RING_GRANTS];
175 unsigned int evtchn, irq;
176 struct work_struct work;
177 struct gnttab_free_callback callback;
178 struct blk_shadow shadow[BLK_MAX_RING_SIZE];
179 struct list_head indirect_pages;
180 struct list_head grants;
181 unsigned int persistent_gnts_c;
182 unsigned long shadow_free;
183 struct blkfront_info *dev_info;
184};
185
186
187
188
189
190
191struct blkfront_info
192{
193 struct mutex mutex;
194 struct xenbus_device *xbdev;
195 struct gendisk *gd;
196 u16 sector_size;
197 unsigned int physical_sector_size;
198 int vdevice;
199 blkif_vdev_t handle;
200 enum blkif_state connected;
201
202 unsigned int nr_ring_pages;
203 struct request_queue *rq;
204 unsigned int feature_flush:1;
205 unsigned int feature_fua:1;
206 unsigned int feature_discard:1;
207 unsigned int feature_secdiscard:1;
208 unsigned int feature_persistent:1;
209 unsigned int discard_granularity;
210 unsigned int discard_alignment;
211
212 unsigned int max_indirect_segments;
213 int is_ready;
214 struct blk_mq_tag_set tag_set;
215 struct blkfront_ring_info *rinfo;
216 unsigned int nr_rings;
217
218 struct list_head requests;
219 struct bio_list bio_list;
220};
221
222static unsigned int nr_minors;
223static unsigned long *minors;
224static DEFINE_SPINLOCK(minor_lock);
225
226#define GRANT_INVALID_REF 0
227
228#define PARTS_PER_DISK 16
229#define PARTS_PER_EXT_DISK 256
230
231#define BLKIF_MAJOR(dev) ((dev)>>8)
232#define BLKIF_MINOR(dev) ((dev) & 0xff)
233
234#define EXT_SHIFT 28
235#define EXTENDED (1<<EXT_SHIFT)
236#define VDEV_IS_EXTENDED(dev) ((dev)&(EXTENDED))
237#define BLKIF_MINOR_EXT(dev) ((dev)&(~EXTENDED))
238#define EMULATED_HD_DISK_MINOR_OFFSET (0)
239#define EMULATED_HD_DISK_NAME_OFFSET (EMULATED_HD_DISK_MINOR_OFFSET / 256)
240#define EMULATED_SD_DISK_MINOR_OFFSET (0)
241#define EMULATED_SD_DISK_NAME_OFFSET (EMULATED_SD_DISK_MINOR_OFFSET / 256)
242
243#define DEV_NAME "xvd"
244
245
246
247
248
249
250#define GRANTS_PER_PSEG (PAGE_SIZE / XEN_PAGE_SIZE)
251
252#define GRANTS_PER_INDIRECT_FRAME \
253 (XEN_PAGE_SIZE / sizeof(struct blkif_request_segment))
254
255#define PSEGS_PER_INDIRECT_FRAME \
256 (GRANTS_INDIRECT_FRAME / GRANTS_PSEGS)
257
258#define INDIRECT_GREFS(_grants) \
259 DIV_ROUND_UP(_grants, GRANTS_PER_INDIRECT_FRAME)
260
261#define GREFS(_psegs) ((_psegs) * GRANTS_PER_PSEG)
262
263static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo);
264static void blkfront_gather_backend_features(struct blkfront_info *info);
265static int negotiate_mq(struct blkfront_info *info);
266
267static int get_id_from_freelist(struct blkfront_ring_info *rinfo)
268{
269 unsigned long free = rinfo->shadow_free;
270
271 BUG_ON(free >= BLK_RING_SIZE(rinfo->dev_info));
272 rinfo->shadow_free = rinfo->shadow[free].req.u.rw.id;
273 rinfo->shadow[free].req.u.rw.id = 0x0fffffee;
274 return free;
275}
276
277static int add_id_to_freelist(struct blkfront_ring_info *rinfo,
278 unsigned long id)
279{
280 if (rinfo->shadow[id].req.u.rw.id != id)
281 return -EINVAL;
282 if (rinfo->shadow[id].request == NULL)
283 return -EINVAL;
284 rinfo->shadow[id].req.u.rw.id = rinfo->shadow_free;
285 rinfo->shadow[id].request = NULL;
286 rinfo->shadow_free = id;
287 return 0;
288}
289
290static int fill_grant_buffer(struct blkfront_ring_info *rinfo, int num)
291{
292 struct blkfront_info *info = rinfo->dev_info;
293 struct page *granted_page;
294 struct grant *gnt_list_entry, *n;
295 int i = 0;
296
297 while (i < num) {
298 gnt_list_entry = kzalloc(sizeof(struct grant), GFP_NOIO);
299 if (!gnt_list_entry)
300 goto out_of_memory;
301
302 if (info->feature_persistent) {
303 granted_page = alloc_page(GFP_NOIO);
304 if (!granted_page) {
305 kfree(gnt_list_entry);
306 goto out_of_memory;
307 }
308 gnt_list_entry->page = granted_page;
309 }
310
311 gnt_list_entry->gref = GRANT_INVALID_REF;
312 list_add(&gnt_list_entry->node, &rinfo->grants);
313 i++;
314 }
315
316 return 0;
317
318out_of_memory:
319 list_for_each_entry_safe(gnt_list_entry, n,
320 &rinfo->grants, node) {
321 list_del(&gnt_list_entry->node);
322 if (info->feature_persistent)
323 __free_page(gnt_list_entry->page);
324 kfree(gnt_list_entry);
325 i--;
326 }
327 BUG_ON(i != 0);
328 return -ENOMEM;
329}
330
331static struct grant *get_free_grant(struct blkfront_ring_info *rinfo)
332{
333 struct grant *gnt_list_entry;
334
335 BUG_ON(list_empty(&rinfo->grants));
336 gnt_list_entry = list_first_entry(&rinfo->grants, struct grant,
337 node);
338 list_del(&gnt_list_entry->node);
339
340 if (gnt_list_entry->gref != GRANT_INVALID_REF)
341 rinfo->persistent_gnts_c--;
342
343 return gnt_list_entry;
344}
345
346static inline void grant_foreign_access(const struct grant *gnt_list_entry,
347 const struct blkfront_info *info)
348{
349 gnttab_page_grant_foreign_access_ref_one(gnt_list_entry->gref,
350 info->xbdev->otherend_id,
351 gnt_list_entry->page,
352 0);
353}
354
355static struct grant *get_grant(grant_ref_t *gref_head,
356 unsigned long gfn,
357 struct blkfront_ring_info *rinfo)
358{
359 struct grant *gnt_list_entry = get_free_grant(rinfo);
360 struct blkfront_info *info = rinfo->dev_info;
361
362 if (gnt_list_entry->gref != GRANT_INVALID_REF)
363 return gnt_list_entry;
364
365
366 gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
367 BUG_ON(gnt_list_entry->gref == -ENOSPC);
368 if (info->feature_persistent)
369 grant_foreign_access(gnt_list_entry, info);
370 else {
371
372 gnttab_grant_foreign_access_ref(gnt_list_entry->gref,
373 info->xbdev->otherend_id,
374 gfn, 0);
375 }
376
377 return gnt_list_entry;
378}
379
380static struct grant *get_indirect_grant(grant_ref_t *gref_head,
381 struct blkfront_ring_info *rinfo)
382{
383 struct grant *gnt_list_entry = get_free_grant(rinfo);
384 struct blkfront_info *info = rinfo->dev_info;
385
386 if (gnt_list_entry->gref != GRANT_INVALID_REF)
387 return gnt_list_entry;
388
389
390 gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
391 BUG_ON(gnt_list_entry->gref == -ENOSPC);
392 if (!info->feature_persistent) {
393 struct page *indirect_page;
394
395
396 BUG_ON(list_empty(&rinfo->indirect_pages));
397 indirect_page = list_first_entry(&rinfo->indirect_pages,
398 struct page, lru);
399 list_del(&indirect_page->lru);
400 gnt_list_entry->page = indirect_page;
401 }
402 grant_foreign_access(gnt_list_entry, info);
403
404 return gnt_list_entry;
405}
406
407static const char *op_name(int op)
408{
409 static const char *const names[] = {
410 [BLKIF_OP_READ] = "read",
411 [BLKIF_OP_WRITE] = "write",
412 [BLKIF_OP_WRITE_BARRIER] = "barrier",
413 [BLKIF_OP_FLUSH_DISKCACHE] = "flush",
414 [BLKIF_OP_DISCARD] = "discard" };
415
416 if (op < 0 || op >= ARRAY_SIZE(names))
417 return "unknown";
418
419 if (!names[op])
420 return "reserved";
421
422 return names[op];
423}
424static int xlbd_reserve_minors(unsigned int minor, unsigned int nr)
425{
426 unsigned int end = minor + nr;
427 int rc;
428
429 if (end > nr_minors) {
430 unsigned long *bitmap, *old;
431
432 bitmap = kcalloc(BITS_TO_LONGS(end), sizeof(*bitmap),
433 GFP_KERNEL);
434 if (bitmap == NULL)
435 return -ENOMEM;
436
437 spin_lock(&minor_lock);
438 if (end > nr_minors) {
439 old = minors;
440 memcpy(bitmap, minors,
441 BITS_TO_LONGS(nr_minors) * sizeof(*bitmap));
442 minors = bitmap;
443 nr_minors = BITS_TO_LONGS(end) * BITS_PER_LONG;
444 } else
445 old = bitmap;
446 spin_unlock(&minor_lock);
447 kfree(old);
448 }
449
450 spin_lock(&minor_lock);
451 if (find_next_bit(minors, end, minor) >= end) {
452 bitmap_set(minors, minor, nr);
453 rc = 0;
454 } else
455 rc = -EBUSY;
456 spin_unlock(&minor_lock);
457
458 return rc;
459}
460
461static void xlbd_release_minors(unsigned int minor, unsigned int nr)
462{
463 unsigned int end = minor + nr;
464
465 BUG_ON(end > nr_minors);
466 spin_lock(&minor_lock);
467 bitmap_clear(minors, minor, nr);
468 spin_unlock(&minor_lock);
469}
470
471static void blkif_restart_queue_callback(void *arg)
472{
473 struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)arg;
474 schedule_work(&rinfo->work);
475}
476
477static int blkif_getgeo(struct block_device *bd, struct hd_geometry *hg)
478{
479
480
481 sector_t nsect = get_capacity(bd->bd_disk);
482 sector_t cylinders = nsect;
483
484 hg->heads = 0xff;
485 hg->sectors = 0x3f;
486 sector_div(cylinders, hg->heads * hg->sectors);
487 hg->cylinders = cylinders;
488 if ((sector_t)(hg->cylinders + 1) * hg->heads * hg->sectors < nsect)
489 hg->cylinders = 0xffff;
490 return 0;
491}
492
493static int blkif_ioctl(struct block_device *bdev, fmode_t mode,
494 unsigned command, unsigned long argument)
495{
496 struct blkfront_info *info = bdev->bd_disk->private_data;
497 int i;
498
499 dev_dbg(&info->xbdev->dev, "command: 0x%x, argument: 0x%lx\n",
500 command, (long)argument);
501
502 switch (command) {
503 case CDROMMULTISESSION:
504 dev_dbg(&info->xbdev->dev, "FIXME: support multisession CDs later\n");
505 for (i = 0; i < sizeof(struct cdrom_multisession); i++)
506 if (put_user(0, (char __user *)(argument + i)))
507 return -EFAULT;
508 return 0;
509
510 case CDROM_GET_CAPABILITY: {
511 struct gendisk *gd = info->gd;
512 if (gd->flags & GENHD_FL_CD)
513 return 0;
514 return -EINVAL;
515 }
516
517 default:
518
519
520 return -EINVAL;
521 }
522
523 return 0;
524}
525
526static unsigned long blkif_ring_get_request(struct blkfront_ring_info *rinfo,
527 struct request *req,
528 struct blkif_request **ring_req)
529{
530 unsigned long id;
531
532 *ring_req = RING_GET_REQUEST(&rinfo->ring, rinfo->ring.req_prod_pvt);
533 rinfo->ring.req_prod_pvt++;
534
535 id = get_id_from_freelist(rinfo);
536 rinfo->shadow[id].request = req;
537 rinfo->shadow[id].status = REQ_WAITING;
538 rinfo->shadow[id].associated_id = NO_ASSOCIATED_ID;
539
540 (*ring_req)->u.rw.id = id;
541
542 return id;
543}
544
545static int blkif_queue_discard_req(struct request *req, struct blkfront_ring_info *rinfo)
546{
547 struct blkfront_info *info = rinfo->dev_info;
548 struct blkif_request *ring_req;
549 unsigned long id;
550
551
552 id = blkif_ring_get_request(rinfo, req, &ring_req);
553
554 ring_req->operation = BLKIF_OP_DISCARD;
555 ring_req->u.discard.nr_sectors = blk_rq_sectors(req);
556 ring_req->u.discard.id = id;
557 ring_req->u.discard.sector_number = (blkif_sector_t)blk_rq_pos(req);
558 if (req_op(req) == REQ_OP_SECURE_ERASE && info->feature_secdiscard)
559 ring_req->u.discard.flag = BLKIF_DISCARD_SECURE;
560 else
561 ring_req->u.discard.flag = 0;
562
563
564 rinfo->shadow[id].req = *ring_req;
565
566 return 0;
567}
568
569struct setup_rw_req {
570 unsigned int grant_idx;
571 struct blkif_request_segment *segments;
572 struct blkfront_ring_info *rinfo;
573 struct blkif_request *ring_req;
574 grant_ref_t gref_head;
575 unsigned int id;
576
577 bool need_copy;
578 unsigned int bvec_off;
579 char *bvec_data;
580
581 bool require_extra_req;
582 struct blkif_request *extra_ring_req;
583};
584
585static void blkif_setup_rw_req_grant(unsigned long gfn, unsigned int offset,
586 unsigned int len, void *data)
587{
588 struct setup_rw_req *setup = data;
589 int n, ref;
590 struct grant *gnt_list_entry;
591 unsigned int fsect, lsect;
592
593 unsigned int grant_idx = setup->grant_idx;
594 struct blkif_request *ring_req = setup->ring_req;
595 struct blkfront_ring_info *rinfo = setup->rinfo;
596
597
598
599
600
601
602 struct blk_shadow *shadow = &rinfo->shadow[setup->id];
603
604 if (unlikely(setup->require_extra_req &&
605 grant_idx >= BLKIF_MAX_SEGMENTS_PER_REQUEST)) {
606
607
608
609
610 grant_idx -= BLKIF_MAX_SEGMENTS_PER_REQUEST;
611 ring_req = setup->extra_ring_req;
612 }
613
614 if ((ring_req->operation == BLKIF_OP_INDIRECT) &&
615 (grant_idx % GRANTS_PER_INDIRECT_FRAME == 0)) {
616 if (setup->segments)
617 kunmap_atomic(setup->segments);
618
619 n = grant_idx / GRANTS_PER_INDIRECT_FRAME;
620 gnt_list_entry = get_indirect_grant(&setup->gref_head, rinfo);
621 shadow->indirect_grants[n] = gnt_list_entry;
622 setup->segments = kmap_atomic(gnt_list_entry->page);
623 ring_req->u.indirect.indirect_grefs[n] = gnt_list_entry->gref;
624 }
625
626 gnt_list_entry = get_grant(&setup->gref_head, gfn, rinfo);
627 ref = gnt_list_entry->gref;
628
629
630
631
632 shadow->grants_used[setup->grant_idx] = gnt_list_entry;
633
634 if (setup->need_copy) {
635 void *shared_data;
636
637 shared_data = kmap_atomic(gnt_list_entry->page);
638
639
640
641
642
643
644
645
646
647 memcpy(shared_data + offset,
648 setup->bvec_data + setup->bvec_off,
649 len);
650
651 kunmap_atomic(shared_data);
652 setup->bvec_off += len;
653 }
654
655 fsect = offset >> 9;
656 lsect = fsect + (len >> 9) - 1;
657 if (ring_req->operation != BLKIF_OP_INDIRECT) {
658 ring_req->u.rw.seg[grant_idx] =
659 (struct blkif_request_segment) {
660 .gref = ref,
661 .first_sect = fsect,
662 .last_sect = lsect };
663 } else {
664 setup->segments[grant_idx % GRANTS_PER_INDIRECT_FRAME] =
665 (struct blkif_request_segment) {
666 .gref = ref,
667 .first_sect = fsect,
668 .last_sect = lsect };
669 }
670
671 (setup->grant_idx)++;
672}
673
674static void blkif_setup_extra_req(struct blkif_request *first,
675 struct blkif_request *second)
676{
677 uint16_t nr_segments = first->u.rw.nr_segments;
678
679
680
681
682
683 first->u.rw.nr_segments = BLKIF_MAX_SEGMENTS_PER_REQUEST;
684
685 second->u.rw.nr_segments = nr_segments - BLKIF_MAX_SEGMENTS_PER_REQUEST;
686 second->u.rw.sector_number = first->u.rw.sector_number +
687 (BLKIF_MAX_SEGMENTS_PER_REQUEST * XEN_PAGE_SIZE) / 512;
688
689 second->u.rw.handle = first->u.rw.handle;
690 second->operation = first->operation;
691}
692
693static int blkif_queue_rw_req(struct request *req, struct blkfront_ring_info *rinfo)
694{
695 struct blkfront_info *info = rinfo->dev_info;
696 struct blkif_request *ring_req, *extra_ring_req = NULL;
697 unsigned long id, extra_id = NO_ASSOCIATED_ID;
698 bool require_extra_req = false;
699 int i;
700 struct setup_rw_req setup = {
701 .grant_idx = 0,
702 .segments = NULL,
703 .rinfo = rinfo,
704 .need_copy = rq_data_dir(req) && info->feature_persistent,
705 };
706
707
708
709
710
711
712 bool new_persistent_gnts = false;
713 struct scatterlist *sg;
714 int num_sg, max_grefs, num_grant;
715
716 max_grefs = req->nr_phys_segments * GRANTS_PER_PSEG;
717 if (max_grefs > BLKIF_MAX_SEGMENTS_PER_REQUEST)
718
719
720
721
722 max_grefs += INDIRECT_GREFS(max_grefs);
723
724
725 if (rinfo->persistent_gnts_c < max_grefs) {
726 new_persistent_gnts = true;
727
728 if (gnttab_alloc_grant_references(
729 max_grefs - rinfo->persistent_gnts_c,
730 &setup.gref_head) < 0) {
731 gnttab_request_free_callback(
732 &rinfo->callback,
733 blkif_restart_queue_callback,
734 rinfo,
735 max_grefs - rinfo->persistent_gnts_c);
736 return 1;
737 }
738 }
739
740
741 id = blkif_ring_get_request(rinfo, req, &ring_req);
742
743 num_sg = blk_rq_map_sg(req->q, req, rinfo->shadow[id].sg);
744 num_grant = 0;
745
746 for_each_sg(rinfo->shadow[id].sg, sg, num_sg, i)
747 num_grant += gnttab_count_grant(sg->offset, sg->length);
748
749 require_extra_req = info->max_indirect_segments == 0 &&
750 num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST;
751 BUG_ON(!HAS_EXTRA_REQ && require_extra_req);
752
753 rinfo->shadow[id].num_sg = num_sg;
754 if (num_grant > BLKIF_MAX_SEGMENTS_PER_REQUEST &&
755 likely(!require_extra_req)) {
756
757
758
759
760 BUG_ON(req_op(req) == REQ_OP_FLUSH || req->cmd_flags & REQ_FUA);
761 ring_req->operation = BLKIF_OP_INDIRECT;
762 ring_req->u.indirect.indirect_op = rq_data_dir(req) ?
763 BLKIF_OP_WRITE : BLKIF_OP_READ;
764 ring_req->u.indirect.sector_number = (blkif_sector_t)blk_rq_pos(req);
765 ring_req->u.indirect.handle = info->handle;
766 ring_req->u.indirect.nr_segments = num_grant;
767 } else {
768 ring_req->u.rw.sector_number = (blkif_sector_t)blk_rq_pos(req);
769 ring_req->u.rw.handle = info->handle;
770 ring_req->operation = rq_data_dir(req) ?
771 BLKIF_OP_WRITE : BLKIF_OP_READ;
772 if (req_op(req) == REQ_OP_FLUSH || req->cmd_flags & REQ_FUA) {
773
774
775
776
777
778
779
780 if (info->feature_flush && info->feature_fua)
781 ring_req->operation =
782 BLKIF_OP_WRITE_BARRIER;
783 else if (info->feature_flush)
784 ring_req->operation =
785 BLKIF_OP_FLUSH_DISKCACHE;
786 else
787 ring_req->operation = 0;
788 }
789 ring_req->u.rw.nr_segments = num_grant;
790 if (unlikely(require_extra_req)) {
791 extra_id = blkif_ring_get_request(rinfo, req,
792 &extra_ring_req);
793
794
795
796
797 rinfo->shadow[extra_id].num_sg = 0;
798
799 blkif_setup_extra_req(ring_req, extra_ring_req);
800
801
802 rinfo->shadow[extra_id].associated_id = id;
803 rinfo->shadow[id].associated_id = extra_id;
804 }
805 }
806
807 setup.ring_req = ring_req;
808 setup.id = id;
809
810 setup.require_extra_req = require_extra_req;
811 if (unlikely(require_extra_req))
812 setup.extra_ring_req = extra_ring_req;
813
814 for_each_sg(rinfo->shadow[id].sg, sg, num_sg, i) {
815 BUG_ON(sg->offset + sg->length > PAGE_SIZE);
816
817 if (setup.need_copy) {
818 setup.bvec_off = sg->offset;
819 setup.bvec_data = kmap_atomic(sg_page(sg));
820 }
821
822 gnttab_foreach_grant_in_range(sg_page(sg),
823 sg->offset,
824 sg->length,
825 blkif_setup_rw_req_grant,
826 &setup);
827
828 if (setup.need_copy)
829 kunmap_atomic(setup.bvec_data);
830 }
831 if (setup.segments)
832 kunmap_atomic(setup.segments);
833
834
835 rinfo->shadow[id].req = *ring_req;
836 if (unlikely(require_extra_req))
837 rinfo->shadow[extra_id].req = *extra_ring_req;
838
839 if (new_persistent_gnts)
840 gnttab_free_grant_references(setup.gref_head);
841
842 return 0;
843}
844
845
846
847
848
849
850
851static int blkif_queue_request(struct request *req, struct blkfront_ring_info *rinfo)
852{
853 if (unlikely(rinfo->dev_info->connected != BLKIF_STATE_CONNECTED))
854 return 1;
855
856 if (unlikely(req_op(req) == REQ_OP_DISCARD ||
857 req_op(req) == REQ_OP_SECURE_ERASE))
858 return blkif_queue_discard_req(req, rinfo);
859 else
860 return blkif_queue_rw_req(req, rinfo);
861}
862
863static inline void flush_requests(struct blkfront_ring_info *rinfo)
864{
865 int notify;
866
867 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&rinfo->ring, notify);
868
869 if (notify)
870 notify_remote_via_irq(rinfo->irq);
871}
872
873static inline bool blkif_request_flush_invalid(struct request *req,
874 struct blkfront_info *info)
875{
876 return (blk_rq_is_passthrough(req) ||
877 ((req_op(req) == REQ_OP_FLUSH) &&
878 !info->feature_flush) ||
879 ((req->cmd_flags & REQ_FUA) &&
880 !info->feature_fua));
881}
882
883static blk_status_t blkif_queue_rq(struct blk_mq_hw_ctx *hctx,
884 const struct blk_mq_queue_data *qd)
885{
886 unsigned long flags;
887 int qid = hctx->queue_num;
888 struct blkfront_info *info = hctx->queue->queuedata;
889 struct blkfront_ring_info *rinfo = NULL;
890
891 BUG_ON(info->nr_rings <= qid);
892 rinfo = &info->rinfo[qid];
893 blk_mq_start_request(qd->rq);
894 spin_lock_irqsave(&rinfo->ring_lock, flags);
895 if (RING_FULL(&rinfo->ring))
896 goto out_busy;
897
898 if (blkif_request_flush_invalid(qd->rq, rinfo->dev_info))
899 goto out_err;
900
901 if (blkif_queue_request(qd->rq, rinfo))
902 goto out_busy;
903
904 flush_requests(rinfo);
905 spin_unlock_irqrestore(&rinfo->ring_lock, flags);
906 return BLK_STS_OK;
907
908out_err:
909 spin_unlock_irqrestore(&rinfo->ring_lock, flags);
910 return BLK_STS_IOERR;
911
912out_busy:
913 blk_mq_stop_hw_queue(hctx);
914 spin_unlock_irqrestore(&rinfo->ring_lock, flags);
915 return BLK_STS_DEV_RESOURCE;
916}
917
918static void blkif_complete_rq(struct request *rq)
919{
920 blk_mq_end_request(rq, blkif_req(rq)->error);
921}
922
923static const struct blk_mq_ops blkfront_mq_ops = {
924 .queue_rq = blkif_queue_rq,
925 .complete = blkif_complete_rq,
926};
927
928static void blkif_set_queue_limits(struct blkfront_info *info)
929{
930 struct request_queue *rq = info->rq;
931 struct gendisk *gd = info->gd;
932 unsigned int segments = info->max_indirect_segments ? :
933 BLKIF_MAX_SEGMENTS_PER_REQUEST;
934
935 queue_flag_set_unlocked(QUEUE_FLAG_VIRT, rq);
936
937 if (info->feature_discard) {
938 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, rq);
939 blk_queue_max_discard_sectors(rq, get_capacity(gd));
940 rq->limits.discard_granularity = info->discard_granularity;
941 rq->limits.discard_alignment = info->discard_alignment;
942 if (info->feature_secdiscard)
943 queue_flag_set_unlocked(QUEUE_FLAG_SECERASE, rq);
944 }
945
946
947 blk_queue_logical_block_size(rq, info->sector_size);
948 blk_queue_physical_block_size(rq, info->physical_sector_size);
949 blk_queue_max_hw_sectors(rq, (segments * XEN_PAGE_SIZE) / 512);
950
951
952 blk_queue_segment_boundary(rq, PAGE_SIZE - 1);
953 blk_queue_max_segment_size(rq, PAGE_SIZE);
954
955
956 blk_queue_max_segments(rq, segments / GRANTS_PER_PSEG);
957
958
959 blk_queue_dma_alignment(rq, 511);
960}
961
962static int xlvbd_init_blk_queue(struct gendisk *gd, u16 sector_size,
963 unsigned int physical_sector_size)
964{
965 struct request_queue *rq;
966 struct blkfront_info *info = gd->private_data;
967
968 memset(&info->tag_set, 0, sizeof(info->tag_set));
969 info->tag_set.ops = &blkfront_mq_ops;
970 info->tag_set.nr_hw_queues = info->nr_rings;
971 if (HAS_EXTRA_REQ && info->max_indirect_segments == 0) {
972
973
974
975
976
977
978 info->tag_set.queue_depth = BLK_RING_SIZE(info) / 2;
979 } else
980 info->tag_set.queue_depth = BLK_RING_SIZE(info);
981 info->tag_set.numa_node = NUMA_NO_NODE;
982 info->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
983 info->tag_set.cmd_size = sizeof(struct blkif_req);
984 info->tag_set.driver_data = info;
985
986 if (blk_mq_alloc_tag_set(&info->tag_set))
987 return -EINVAL;
988 rq = blk_mq_init_queue(&info->tag_set);
989 if (IS_ERR(rq)) {
990 blk_mq_free_tag_set(&info->tag_set);
991 return PTR_ERR(rq);
992 }
993
994 rq->queuedata = info;
995 info->rq = gd->queue = rq;
996 info->gd = gd;
997 info->sector_size = sector_size;
998 info->physical_sector_size = physical_sector_size;
999 blkif_set_queue_limits(info);
1000
1001 return 0;
1002}
1003
1004static const char *flush_info(struct blkfront_info *info)
1005{
1006 if (info->feature_flush && info->feature_fua)
1007 return "barrier: enabled;";
1008 else if (info->feature_flush)
1009 return "flush diskcache: enabled;";
1010 else
1011 return "barrier or flush: disabled;";
1012}
1013
1014static void xlvbd_flush(struct blkfront_info *info)
1015{
1016 blk_queue_write_cache(info->rq, info->feature_flush ? true : false,
1017 info->feature_fua ? true : false);
1018 pr_info("blkfront: %s: %s %s %s %s %s\n",
1019 info->gd->disk_name, flush_info(info),
1020 "persistent grants:", info->feature_persistent ?
1021 "enabled;" : "disabled;", "indirect descriptors:",
1022 info->max_indirect_segments ? "enabled;" : "disabled;");
1023}
1024
1025static int xen_translate_vdev(int vdevice, int *minor, unsigned int *offset)
1026{
1027 int major;
1028 major = BLKIF_MAJOR(vdevice);
1029 *minor = BLKIF_MINOR(vdevice);
1030 switch (major) {
1031 case XEN_IDE0_MAJOR:
1032 *offset = (*minor / 64) + EMULATED_HD_DISK_NAME_OFFSET;
1033 *minor = ((*minor / 64) * PARTS_PER_DISK) +
1034 EMULATED_HD_DISK_MINOR_OFFSET;
1035 break;
1036 case XEN_IDE1_MAJOR:
1037 *offset = (*minor / 64) + 2 + EMULATED_HD_DISK_NAME_OFFSET;
1038 *minor = (((*minor / 64) + 2) * PARTS_PER_DISK) +
1039 EMULATED_HD_DISK_MINOR_OFFSET;
1040 break;
1041 case XEN_SCSI_DISK0_MAJOR:
1042 *offset = (*minor / PARTS_PER_DISK) + EMULATED_SD_DISK_NAME_OFFSET;
1043 *minor = *minor + EMULATED_SD_DISK_MINOR_OFFSET;
1044 break;
1045 case XEN_SCSI_DISK1_MAJOR:
1046 case XEN_SCSI_DISK2_MAJOR:
1047 case XEN_SCSI_DISK3_MAJOR:
1048 case XEN_SCSI_DISK4_MAJOR:
1049 case XEN_SCSI_DISK5_MAJOR:
1050 case XEN_SCSI_DISK6_MAJOR:
1051 case XEN_SCSI_DISK7_MAJOR:
1052 *offset = (*minor / PARTS_PER_DISK) +
1053 ((major - XEN_SCSI_DISK1_MAJOR + 1) * 16) +
1054 EMULATED_SD_DISK_NAME_OFFSET;
1055 *minor = *minor +
1056 ((major - XEN_SCSI_DISK1_MAJOR + 1) * 16 * PARTS_PER_DISK) +
1057 EMULATED_SD_DISK_MINOR_OFFSET;
1058 break;
1059 case XEN_SCSI_DISK8_MAJOR:
1060 case XEN_SCSI_DISK9_MAJOR:
1061 case XEN_SCSI_DISK10_MAJOR:
1062 case XEN_SCSI_DISK11_MAJOR:
1063 case XEN_SCSI_DISK12_MAJOR:
1064 case XEN_SCSI_DISK13_MAJOR:
1065 case XEN_SCSI_DISK14_MAJOR:
1066 case XEN_SCSI_DISK15_MAJOR:
1067 *offset = (*minor / PARTS_PER_DISK) +
1068 ((major - XEN_SCSI_DISK8_MAJOR + 8) * 16) +
1069 EMULATED_SD_DISK_NAME_OFFSET;
1070 *minor = *minor +
1071 ((major - XEN_SCSI_DISK8_MAJOR + 8) * 16 * PARTS_PER_DISK) +
1072 EMULATED_SD_DISK_MINOR_OFFSET;
1073 break;
1074 case XENVBD_MAJOR:
1075 *offset = *minor / PARTS_PER_DISK;
1076 break;
1077 default:
1078 printk(KERN_WARNING "blkfront: your disk configuration is "
1079 "incorrect, please use an xvd device instead\n");
1080 return -ENODEV;
1081 }
1082 return 0;
1083}
1084
1085static char *encode_disk_name(char *ptr, unsigned int n)
1086{
1087 if (n >= 26)
1088 ptr = encode_disk_name(ptr, n / 26 - 1);
1089 *ptr = 'a' + n % 26;
1090 return ptr + 1;
1091}
1092
1093static int xlvbd_alloc_gendisk(blkif_sector_t capacity,
1094 struct blkfront_info *info,
1095 u16 vdisk_info, u16 sector_size,
1096 unsigned int physical_sector_size)
1097{
1098 struct gendisk *gd;
1099 int nr_minors = 1;
1100 int err;
1101 unsigned int offset;
1102 int minor;
1103 int nr_parts;
1104 char *ptr;
1105
1106 BUG_ON(info->gd != NULL);
1107 BUG_ON(info->rq != NULL);
1108
1109 if ((info->vdevice>>EXT_SHIFT) > 1) {
1110
1111 printk(KERN_WARNING "blkfront: vdevice 0x%x is above the extended range; ignoring\n", info->vdevice);
1112 return -ENODEV;
1113 }
1114
1115 if (!VDEV_IS_EXTENDED(info->vdevice)) {
1116 err = xen_translate_vdev(info->vdevice, &minor, &offset);
1117 if (err)
1118 return err;
1119 nr_parts = PARTS_PER_DISK;
1120 } else {
1121 minor = BLKIF_MINOR_EXT(info->vdevice);
1122 nr_parts = PARTS_PER_EXT_DISK;
1123 offset = minor / nr_parts;
1124 if (xen_hvm_domain() && offset < EMULATED_HD_DISK_NAME_OFFSET + 4)
1125 printk(KERN_WARNING "blkfront: vdevice 0x%x might conflict with "
1126 "emulated IDE disks,\n\t choose an xvd device name"
1127 "from xvde on\n", info->vdevice);
1128 }
1129 if (minor >> MINORBITS) {
1130 pr_warn("blkfront: %#x's minor (%#x) out of range; ignoring\n",
1131 info->vdevice, minor);
1132 return -ENODEV;
1133 }
1134
1135 if ((minor % nr_parts) == 0)
1136 nr_minors = nr_parts;
1137
1138 err = xlbd_reserve_minors(minor, nr_minors);
1139 if (err)
1140 goto out;
1141 err = -ENODEV;
1142
1143 gd = alloc_disk(nr_minors);
1144 if (gd == NULL)
1145 goto release;
1146
1147 strcpy(gd->disk_name, DEV_NAME);
1148 ptr = encode_disk_name(gd->disk_name + sizeof(DEV_NAME) - 1, offset);
1149 BUG_ON(ptr >= gd->disk_name + DISK_NAME_LEN);
1150 if (nr_minors > 1)
1151 *ptr = 0;
1152 else
1153 snprintf(ptr, gd->disk_name + DISK_NAME_LEN - ptr,
1154 "%d", minor & (nr_parts - 1));
1155
1156 gd->major = XENVBD_MAJOR;
1157 gd->first_minor = minor;
1158 gd->fops = &xlvbd_block_fops;
1159 gd->private_data = info;
1160 set_capacity(gd, capacity);
1161
1162 if (xlvbd_init_blk_queue(gd, sector_size, physical_sector_size)) {
1163 del_gendisk(gd);
1164 goto release;
1165 }
1166
1167 xlvbd_flush(info);
1168
1169 if (vdisk_info & VDISK_READONLY)
1170 set_disk_ro(gd, 1);
1171
1172 if (vdisk_info & VDISK_REMOVABLE)
1173 gd->flags |= GENHD_FL_REMOVABLE;
1174
1175 if (vdisk_info & VDISK_CDROM)
1176 gd->flags |= GENHD_FL_CD;
1177
1178 return 0;
1179
1180 release:
1181 xlbd_release_minors(minor, nr_minors);
1182 out:
1183 return err;
1184}
1185
1186static void xlvbd_release_gendisk(struct blkfront_info *info)
1187{
1188 unsigned int minor, nr_minors, i;
1189
1190 if (info->rq == NULL)
1191 return;
1192
1193
1194 blk_mq_stop_hw_queues(info->rq);
1195
1196 for (i = 0; i < info->nr_rings; i++) {
1197 struct blkfront_ring_info *rinfo = &info->rinfo[i];
1198
1199
1200 gnttab_cancel_free_callback(&rinfo->callback);
1201
1202
1203 flush_work(&rinfo->work);
1204 }
1205
1206 del_gendisk(info->gd);
1207
1208 minor = info->gd->first_minor;
1209 nr_minors = info->gd->minors;
1210 xlbd_release_minors(minor, nr_minors);
1211
1212 blk_cleanup_queue(info->rq);
1213 blk_mq_free_tag_set(&info->tag_set);
1214 info->rq = NULL;
1215
1216 put_disk(info->gd);
1217 info->gd = NULL;
1218}
1219
1220
1221static inline void kick_pending_request_queues_locked(struct blkfront_ring_info *rinfo)
1222{
1223 if (!RING_FULL(&rinfo->ring))
1224 blk_mq_start_stopped_hw_queues(rinfo->dev_info->rq, true);
1225}
1226
1227static void kick_pending_request_queues(struct blkfront_ring_info *rinfo)
1228{
1229 unsigned long flags;
1230
1231 spin_lock_irqsave(&rinfo->ring_lock, flags);
1232 kick_pending_request_queues_locked(rinfo);
1233 spin_unlock_irqrestore(&rinfo->ring_lock, flags);
1234}
1235
1236static void blkif_restart_queue(struct work_struct *work)
1237{
1238 struct blkfront_ring_info *rinfo = container_of(work, struct blkfront_ring_info, work);
1239
1240 if (rinfo->dev_info->connected == BLKIF_STATE_CONNECTED)
1241 kick_pending_request_queues(rinfo);
1242}
1243
1244static void blkif_free_ring(struct blkfront_ring_info *rinfo)
1245{
1246 struct grant *persistent_gnt, *n;
1247 struct blkfront_info *info = rinfo->dev_info;
1248 int i, j, segs;
1249
1250
1251
1252
1253
1254 if (!list_empty(&rinfo->indirect_pages)) {
1255 struct page *indirect_page, *n;
1256
1257 BUG_ON(info->feature_persistent);
1258 list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) {
1259 list_del(&indirect_page->lru);
1260 __free_page(indirect_page);
1261 }
1262 }
1263
1264
1265 if (!list_empty(&rinfo->grants)) {
1266 list_for_each_entry_safe(persistent_gnt, n,
1267 &rinfo->grants, node) {
1268 list_del(&persistent_gnt->node);
1269 if (persistent_gnt->gref != GRANT_INVALID_REF) {
1270 gnttab_end_foreign_access(persistent_gnt->gref,
1271 0, 0UL);
1272 rinfo->persistent_gnts_c--;
1273 }
1274 if (info->feature_persistent)
1275 __free_page(persistent_gnt->page);
1276 kfree(persistent_gnt);
1277 }
1278 }
1279 BUG_ON(rinfo->persistent_gnts_c != 0);
1280
1281 for (i = 0; i < BLK_RING_SIZE(info); i++) {
1282
1283
1284
1285
1286 if (!rinfo->shadow[i].request)
1287 goto free_shadow;
1288
1289 segs = rinfo->shadow[i].req.operation == BLKIF_OP_INDIRECT ?
1290 rinfo->shadow[i].req.u.indirect.nr_segments :
1291 rinfo->shadow[i].req.u.rw.nr_segments;
1292 for (j = 0; j < segs; j++) {
1293 persistent_gnt = rinfo->shadow[i].grants_used[j];
1294 gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
1295 if (info->feature_persistent)
1296 __free_page(persistent_gnt->page);
1297 kfree(persistent_gnt);
1298 }
1299
1300 if (rinfo->shadow[i].req.operation != BLKIF_OP_INDIRECT)
1301
1302
1303
1304
1305 goto free_shadow;
1306
1307 for (j = 0; j < INDIRECT_GREFS(segs); j++) {
1308 persistent_gnt = rinfo->shadow[i].indirect_grants[j];
1309 gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
1310 __free_page(persistent_gnt->page);
1311 kfree(persistent_gnt);
1312 }
1313
1314free_shadow:
1315 kfree(rinfo->shadow[i].grants_used);
1316 rinfo->shadow[i].grants_used = NULL;
1317 kfree(rinfo->shadow[i].indirect_grants);
1318 rinfo->shadow[i].indirect_grants = NULL;
1319 kfree(rinfo->shadow[i].sg);
1320 rinfo->shadow[i].sg = NULL;
1321 }
1322
1323
1324 gnttab_cancel_free_callback(&rinfo->callback);
1325
1326
1327 flush_work(&rinfo->work);
1328
1329
1330 for (i = 0; i < info->nr_ring_pages; i++) {
1331 if (rinfo->ring_ref[i] != GRANT_INVALID_REF) {
1332 gnttab_end_foreign_access(rinfo->ring_ref[i], 0, 0);
1333 rinfo->ring_ref[i] = GRANT_INVALID_REF;
1334 }
1335 }
1336 free_pages((unsigned long)rinfo->ring.sring, get_order(info->nr_ring_pages * XEN_PAGE_SIZE));
1337 rinfo->ring.sring = NULL;
1338
1339 if (rinfo->irq)
1340 unbind_from_irqhandler(rinfo->irq, rinfo);
1341 rinfo->evtchn = rinfo->irq = 0;
1342}
1343
1344static void blkif_free(struct blkfront_info *info, int suspend)
1345{
1346 unsigned int i;
1347
1348
1349 info->connected = suspend ?
1350 BLKIF_STATE_SUSPENDED : BLKIF_STATE_DISCONNECTED;
1351
1352 if (info->rq)
1353 blk_mq_stop_hw_queues(info->rq);
1354
1355 for (i = 0; i < info->nr_rings; i++)
1356 blkif_free_ring(&info->rinfo[i]);
1357
1358 kfree(info->rinfo);
1359 info->rinfo = NULL;
1360 info->nr_rings = 0;
1361}
1362
1363struct copy_from_grant {
1364 const struct blk_shadow *s;
1365 unsigned int grant_idx;
1366 unsigned int bvec_offset;
1367 char *bvec_data;
1368};
1369
1370static void blkif_copy_from_grant(unsigned long gfn, unsigned int offset,
1371 unsigned int len, void *data)
1372{
1373 struct copy_from_grant *info = data;
1374 char *shared_data;
1375
1376 const struct blk_shadow *s = info->s;
1377
1378 shared_data = kmap_atomic(s->grants_used[info->grant_idx]->page);
1379
1380 memcpy(info->bvec_data + info->bvec_offset,
1381 shared_data + offset, len);
1382
1383 info->bvec_offset += len;
1384 info->grant_idx++;
1385
1386 kunmap_atomic(shared_data);
1387}
1388
1389static enum blk_req_status blkif_rsp_to_req_status(int rsp)
1390{
1391 switch (rsp)
1392 {
1393 case BLKIF_RSP_OKAY:
1394 return REQ_DONE;
1395 case BLKIF_RSP_EOPNOTSUPP:
1396 return REQ_EOPNOTSUPP;
1397 case BLKIF_RSP_ERROR:
1398
1399 default:
1400 return REQ_ERROR;
1401 }
1402}
1403
1404
1405
1406
1407static int blkif_get_final_status(enum blk_req_status s1,
1408 enum blk_req_status s2)
1409{
1410 BUG_ON(s1 == REQ_WAITING);
1411 BUG_ON(s2 == REQ_WAITING);
1412
1413 if (s1 == REQ_ERROR || s2 == REQ_ERROR)
1414 return BLKIF_RSP_ERROR;
1415 else if (s1 == REQ_EOPNOTSUPP || s2 == REQ_EOPNOTSUPP)
1416 return BLKIF_RSP_EOPNOTSUPP;
1417 return BLKIF_RSP_OKAY;
1418}
1419
1420static bool blkif_completion(unsigned long *id,
1421 struct blkfront_ring_info *rinfo,
1422 struct blkif_response *bret)
1423{
1424 int i = 0;
1425 struct scatterlist *sg;
1426 int num_sg, num_grant;
1427 struct blkfront_info *info = rinfo->dev_info;
1428 struct blk_shadow *s = &rinfo->shadow[*id];
1429 struct copy_from_grant data = {
1430 .grant_idx = 0,
1431 };
1432
1433 num_grant = s->req.operation == BLKIF_OP_INDIRECT ?
1434 s->req.u.indirect.nr_segments : s->req.u.rw.nr_segments;
1435
1436
1437 if (unlikely(s->associated_id != NO_ASSOCIATED_ID)) {
1438 struct blk_shadow *s2 = &rinfo->shadow[s->associated_id];
1439
1440
1441 s->status = blkif_rsp_to_req_status(bret->status);
1442
1443
1444 if (s2->status == REQ_WAITING)
1445 return 0;
1446
1447 bret->status = blkif_get_final_status(s->status,
1448 s2->status);
1449
1450
1451
1452
1453
1454 num_grant += s2->req.u.rw.nr_segments;
1455
1456
1457
1458
1459
1460 if (s2->num_sg != 0) {
1461
1462 *id = s->associated_id;
1463 s = s2;
1464 }
1465
1466
1467
1468
1469
1470 if (add_id_to_freelist(rinfo, s->associated_id))
1471 WARN(1, "%s: can't recycle the second part (id = %ld) of the request\n",
1472 info->gd->disk_name, s->associated_id);
1473 }
1474
1475 data.s = s;
1476 num_sg = s->num_sg;
1477
1478 if (bret->operation == BLKIF_OP_READ && info->feature_persistent) {
1479 for_each_sg(s->sg, sg, num_sg, i) {
1480 BUG_ON(sg->offset + sg->length > PAGE_SIZE);
1481
1482 data.bvec_offset = sg->offset;
1483 data.bvec_data = kmap_atomic(sg_page(sg));
1484
1485 gnttab_foreach_grant_in_range(sg_page(sg),
1486 sg->offset,
1487 sg->length,
1488 blkif_copy_from_grant,
1489 &data);
1490
1491 kunmap_atomic(data.bvec_data);
1492 }
1493 }
1494
1495 for (i = 0; i < num_grant; i++) {
1496 if (gnttab_query_foreign_access(s->grants_used[i]->gref)) {
1497
1498
1499
1500
1501
1502
1503 if (!info->feature_persistent)
1504 pr_alert_ratelimited("backed has not unmapped grant: %u\n",
1505 s->grants_used[i]->gref);
1506 list_add(&s->grants_used[i]->node, &rinfo->grants);
1507 rinfo->persistent_gnts_c++;
1508 } else {
1509
1510
1511
1512
1513
1514
1515 gnttab_end_foreign_access(s->grants_used[i]->gref, 0, 0UL);
1516 s->grants_used[i]->gref = GRANT_INVALID_REF;
1517 list_add_tail(&s->grants_used[i]->node, &rinfo->grants);
1518 }
1519 }
1520 if (s->req.operation == BLKIF_OP_INDIRECT) {
1521 for (i = 0; i < INDIRECT_GREFS(num_grant); i++) {
1522 if (gnttab_query_foreign_access(s->indirect_grants[i]->gref)) {
1523 if (!info->feature_persistent)
1524 pr_alert_ratelimited("backed has not unmapped grant: %u\n",
1525 s->indirect_grants[i]->gref);
1526 list_add(&s->indirect_grants[i]->node, &rinfo->grants);
1527 rinfo->persistent_gnts_c++;
1528 } else {
1529 struct page *indirect_page;
1530
1531 gnttab_end_foreign_access(s->indirect_grants[i]->gref, 0, 0UL);
1532
1533
1534
1535
1536 if (!info->feature_persistent) {
1537 indirect_page = s->indirect_grants[i]->page;
1538 list_add(&indirect_page->lru, &rinfo->indirect_pages);
1539 }
1540 s->indirect_grants[i]->gref = GRANT_INVALID_REF;
1541 list_add_tail(&s->indirect_grants[i]->node, &rinfo->grants);
1542 }
1543 }
1544 }
1545
1546 return 1;
1547}
1548
1549static irqreturn_t blkif_interrupt(int irq, void *dev_id)
1550{
1551 struct request *req;
1552 struct blkif_response *bret;
1553 RING_IDX i, rp;
1554 unsigned long flags;
1555 struct blkfront_ring_info *rinfo = (struct blkfront_ring_info *)dev_id;
1556 struct blkfront_info *info = rinfo->dev_info;
1557
1558 if (unlikely(info->connected != BLKIF_STATE_CONNECTED))
1559 return IRQ_HANDLED;
1560
1561 spin_lock_irqsave(&rinfo->ring_lock, flags);
1562 again:
1563 rp = rinfo->ring.sring->rsp_prod;
1564 rmb();
1565
1566 for (i = rinfo->ring.rsp_cons; i != rp; i++) {
1567 unsigned long id;
1568
1569 bret = RING_GET_RESPONSE(&rinfo->ring, i);
1570 id = bret->id;
1571
1572
1573
1574
1575
1576 if (id >= BLK_RING_SIZE(info)) {
1577 WARN(1, "%s: response to %s has incorrect id (%ld)\n",
1578 info->gd->disk_name, op_name(bret->operation), id);
1579
1580
1581 continue;
1582 }
1583 req = rinfo->shadow[id].request;
1584
1585 if (bret->operation != BLKIF_OP_DISCARD) {
1586
1587
1588
1589
1590 if (!blkif_completion(&id, rinfo, bret))
1591 continue;
1592 }
1593
1594 if (add_id_to_freelist(rinfo, id)) {
1595 WARN(1, "%s: response to %s (id %ld) couldn't be recycled!\n",
1596 info->gd->disk_name, op_name(bret->operation), id);
1597 continue;
1598 }
1599
1600 if (bret->status == BLKIF_RSP_OKAY)
1601 blkif_req(req)->error = BLK_STS_OK;
1602 else
1603 blkif_req(req)->error = BLK_STS_IOERR;
1604
1605 switch (bret->operation) {
1606 case BLKIF_OP_DISCARD:
1607 if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
1608 struct request_queue *rq = info->rq;
1609 printk(KERN_WARNING "blkfront: %s: %s op failed\n",
1610 info->gd->disk_name, op_name(bret->operation));
1611 blkif_req(req)->error = BLK_STS_NOTSUPP;
1612 info->feature_discard = 0;
1613 info->feature_secdiscard = 0;
1614 queue_flag_clear(QUEUE_FLAG_DISCARD, rq);
1615 queue_flag_clear(QUEUE_FLAG_SECERASE, rq);
1616 }
1617 break;
1618 case BLKIF_OP_FLUSH_DISKCACHE:
1619 case BLKIF_OP_WRITE_BARRIER:
1620 if (unlikely(bret->status == BLKIF_RSP_EOPNOTSUPP)) {
1621 printk(KERN_WARNING "blkfront: %s: %s op failed\n",
1622 info->gd->disk_name, op_name(bret->operation));
1623 blkif_req(req)->error = BLK_STS_NOTSUPP;
1624 }
1625 if (unlikely(bret->status == BLKIF_RSP_ERROR &&
1626 rinfo->shadow[id].req.u.rw.nr_segments == 0)) {
1627 printk(KERN_WARNING "blkfront: %s: empty %s op failed\n",
1628 info->gd->disk_name, op_name(bret->operation));
1629 blkif_req(req)->error = BLK_STS_NOTSUPP;
1630 }
1631 if (unlikely(blkif_req(req)->error)) {
1632 if (blkif_req(req)->error == BLK_STS_NOTSUPP)
1633 blkif_req(req)->error = BLK_STS_OK;
1634 info->feature_fua = 0;
1635 info->feature_flush = 0;
1636 xlvbd_flush(info);
1637 }
1638
1639 case BLKIF_OP_READ:
1640 case BLKIF_OP_WRITE:
1641 if (unlikely(bret->status != BLKIF_RSP_OKAY))
1642 dev_dbg(&info->xbdev->dev, "Bad return from blkdev data "
1643 "request: %x\n", bret->status);
1644
1645 break;
1646 default:
1647 BUG();
1648 }
1649
1650 blk_mq_complete_request(req);
1651 }
1652
1653 rinfo->ring.rsp_cons = i;
1654
1655 if (i != rinfo->ring.req_prod_pvt) {
1656 int more_to_do;
1657 RING_FINAL_CHECK_FOR_RESPONSES(&rinfo->ring, more_to_do);
1658 if (more_to_do)
1659 goto again;
1660 } else
1661 rinfo->ring.sring->rsp_event = i + 1;
1662
1663 kick_pending_request_queues_locked(rinfo);
1664
1665 spin_unlock_irqrestore(&rinfo->ring_lock, flags);
1666
1667 return IRQ_HANDLED;
1668}
1669
1670
1671static int setup_blkring(struct xenbus_device *dev,
1672 struct blkfront_ring_info *rinfo)
1673{
1674 struct blkif_sring *sring;
1675 int err, i;
1676 struct blkfront_info *info = rinfo->dev_info;
1677 unsigned long ring_size = info->nr_ring_pages * XEN_PAGE_SIZE;
1678 grant_ref_t gref[XENBUS_MAX_RING_GRANTS];
1679
1680 for (i = 0; i < info->nr_ring_pages; i++)
1681 rinfo->ring_ref[i] = GRANT_INVALID_REF;
1682
1683 sring = (struct blkif_sring *)__get_free_pages(GFP_NOIO | __GFP_HIGH,
1684 get_order(ring_size));
1685 if (!sring) {
1686 xenbus_dev_fatal(dev, -ENOMEM, "allocating shared ring");
1687 return -ENOMEM;
1688 }
1689 SHARED_RING_INIT(sring);
1690 FRONT_RING_INIT(&rinfo->ring, sring, ring_size);
1691
1692 err = xenbus_grant_ring(dev, rinfo->ring.sring, info->nr_ring_pages, gref);
1693 if (err < 0) {
1694 free_pages((unsigned long)sring, get_order(ring_size));
1695 rinfo->ring.sring = NULL;
1696 goto fail;
1697 }
1698 for (i = 0; i < info->nr_ring_pages; i++)
1699 rinfo->ring_ref[i] = gref[i];
1700
1701 err = xenbus_alloc_evtchn(dev, &rinfo->evtchn);
1702 if (err)
1703 goto fail;
1704
1705 err = bind_evtchn_to_irqhandler(rinfo->evtchn, blkif_interrupt, 0,
1706 "blkif", rinfo);
1707 if (err <= 0) {
1708 xenbus_dev_fatal(dev, err,
1709 "bind_evtchn_to_irqhandler failed");
1710 goto fail;
1711 }
1712 rinfo->irq = err;
1713
1714 return 0;
1715fail:
1716 blkif_free(info, 0);
1717 return err;
1718}
1719
1720
1721
1722
1723
1724static int write_per_ring_nodes(struct xenbus_transaction xbt,
1725 struct blkfront_ring_info *rinfo, const char *dir)
1726{
1727 int err;
1728 unsigned int i;
1729 const char *message = NULL;
1730 struct blkfront_info *info = rinfo->dev_info;
1731
1732 if (info->nr_ring_pages == 1) {
1733 err = xenbus_printf(xbt, dir, "ring-ref", "%u", rinfo->ring_ref[0]);
1734 if (err) {
1735 message = "writing ring-ref";
1736 goto abort_transaction;
1737 }
1738 } else {
1739 for (i = 0; i < info->nr_ring_pages; i++) {
1740 char ring_ref_name[RINGREF_NAME_LEN];
1741
1742 snprintf(ring_ref_name, RINGREF_NAME_LEN, "ring-ref%u", i);
1743 err = xenbus_printf(xbt, dir, ring_ref_name,
1744 "%u", rinfo->ring_ref[i]);
1745 if (err) {
1746 message = "writing ring-ref";
1747 goto abort_transaction;
1748 }
1749 }
1750 }
1751
1752 err = xenbus_printf(xbt, dir, "event-channel", "%u", rinfo->evtchn);
1753 if (err) {
1754 message = "writing event-channel";
1755 goto abort_transaction;
1756 }
1757
1758 return 0;
1759
1760abort_transaction:
1761 xenbus_transaction_end(xbt, 1);
1762 if (message)
1763 xenbus_dev_fatal(info->xbdev, err, "%s", message);
1764
1765 return err;
1766}
1767
1768
1769static int talk_to_blkback(struct xenbus_device *dev,
1770 struct blkfront_info *info)
1771{
1772 const char *message = NULL;
1773 struct xenbus_transaction xbt;
1774 int err;
1775 unsigned int i, max_page_order;
1776 unsigned int ring_page_order;
1777
1778 if (!info)
1779 return -ENODEV;
1780
1781 max_page_order = xenbus_read_unsigned(info->xbdev->otherend,
1782 "max-ring-page-order", 0);
1783 ring_page_order = min(xen_blkif_max_ring_order, max_page_order);
1784 info->nr_ring_pages = 1 << ring_page_order;
1785
1786 err = negotiate_mq(info);
1787 if (err)
1788 goto destroy_blkring;
1789
1790 for (i = 0; i < info->nr_rings; i++) {
1791 struct blkfront_ring_info *rinfo = &info->rinfo[i];
1792
1793
1794 err = setup_blkring(dev, rinfo);
1795 if (err)
1796 goto destroy_blkring;
1797 }
1798
1799again:
1800 err = xenbus_transaction_start(&xbt);
1801 if (err) {
1802 xenbus_dev_fatal(dev, err, "starting transaction");
1803 goto destroy_blkring;
1804 }
1805
1806 if (info->nr_ring_pages > 1) {
1807 err = xenbus_printf(xbt, dev->nodename, "ring-page-order", "%u",
1808 ring_page_order);
1809 if (err) {
1810 message = "writing ring-page-order";
1811 goto abort_transaction;
1812 }
1813 }
1814
1815
1816 if (info->nr_rings == 1) {
1817 err = write_per_ring_nodes(xbt, &info->rinfo[0], dev->nodename);
1818 if (err)
1819 goto destroy_blkring;
1820 } else {
1821 char *path;
1822 size_t pathsize;
1823
1824 err = xenbus_printf(xbt, dev->nodename, "multi-queue-num-queues", "%u",
1825 info->nr_rings);
1826 if (err) {
1827 message = "writing multi-queue-num-queues";
1828 goto abort_transaction;
1829 }
1830
1831 pathsize = strlen(dev->nodename) + QUEUE_NAME_LEN;
1832 path = kmalloc(pathsize, GFP_KERNEL);
1833 if (!path) {
1834 err = -ENOMEM;
1835 message = "ENOMEM while writing ring references";
1836 goto abort_transaction;
1837 }
1838
1839 for (i = 0; i < info->nr_rings; i++) {
1840 memset(path, 0, pathsize);
1841 snprintf(path, pathsize, "%s/queue-%u", dev->nodename, i);
1842 err = write_per_ring_nodes(xbt, &info->rinfo[i], path);
1843 if (err) {
1844 kfree(path);
1845 goto destroy_blkring;
1846 }
1847 }
1848 kfree(path);
1849 }
1850 err = xenbus_printf(xbt, dev->nodename, "protocol", "%s",
1851 XEN_IO_PROTO_ABI_NATIVE);
1852 if (err) {
1853 message = "writing protocol";
1854 goto abort_transaction;
1855 }
1856 err = xenbus_printf(xbt, dev->nodename,
1857 "feature-persistent", "%u", 1);
1858 if (err)
1859 dev_warn(&dev->dev,
1860 "writing persistent grants feature to xenbus");
1861
1862 err = xenbus_transaction_end(xbt, 0);
1863 if (err) {
1864 if (err == -EAGAIN)
1865 goto again;
1866 xenbus_dev_fatal(dev, err, "completing transaction");
1867 goto destroy_blkring;
1868 }
1869
1870 for (i = 0; i < info->nr_rings; i++) {
1871 unsigned int j;
1872 struct blkfront_ring_info *rinfo = &info->rinfo[i];
1873
1874 for (j = 0; j < BLK_RING_SIZE(info); j++)
1875 rinfo->shadow[j].req.u.rw.id = j + 1;
1876 rinfo->shadow[BLK_RING_SIZE(info)-1].req.u.rw.id = 0x0fffffff;
1877 }
1878 xenbus_switch_state(dev, XenbusStateInitialised);
1879
1880 return 0;
1881
1882 abort_transaction:
1883 xenbus_transaction_end(xbt, 1);
1884 if (message)
1885 xenbus_dev_fatal(dev, err, "%s", message);
1886 destroy_blkring:
1887 blkif_free(info, 0);
1888
1889 kfree(info);
1890 dev_set_drvdata(&dev->dev, NULL);
1891
1892 return err;
1893}
1894
1895static int negotiate_mq(struct blkfront_info *info)
1896{
1897 unsigned int backend_max_queues;
1898 unsigned int i;
1899
1900 BUG_ON(info->nr_rings);
1901
1902
1903 backend_max_queues = xenbus_read_unsigned(info->xbdev->otherend,
1904 "multi-queue-max-queues", 1);
1905 info->nr_rings = min(backend_max_queues, xen_blkif_max_queues);
1906
1907 if (!info->nr_rings)
1908 info->nr_rings = 1;
1909
1910 info->rinfo = kzalloc(sizeof(struct blkfront_ring_info) * info->nr_rings, GFP_KERNEL);
1911 if (!info->rinfo) {
1912 xenbus_dev_fatal(info->xbdev, -ENOMEM, "allocating ring_info structure");
1913 return -ENOMEM;
1914 }
1915
1916 for (i = 0; i < info->nr_rings; i++) {
1917 struct blkfront_ring_info *rinfo;
1918
1919 rinfo = &info->rinfo[i];
1920 INIT_LIST_HEAD(&rinfo->indirect_pages);
1921 INIT_LIST_HEAD(&rinfo->grants);
1922 rinfo->dev_info = info;
1923 INIT_WORK(&rinfo->work, blkif_restart_queue);
1924 spin_lock_init(&rinfo->ring_lock);
1925 }
1926 return 0;
1927}
1928
1929
1930
1931
1932
1933
1934static int blkfront_probe(struct xenbus_device *dev,
1935 const struct xenbus_device_id *id)
1936{
1937 int err, vdevice;
1938 struct blkfront_info *info;
1939
1940
1941 err = xenbus_scanf(XBT_NIL, dev->nodename,
1942 "virtual-device", "%i", &vdevice);
1943 if (err != 1) {
1944
1945 err = xenbus_scanf(XBT_NIL, dev->nodename, "virtual-device-ext",
1946 "%i", &vdevice);
1947 if (err != 1) {
1948 xenbus_dev_fatal(dev, err, "reading virtual-device");
1949 return err;
1950 }
1951 }
1952
1953 if (xen_hvm_domain()) {
1954 char *type;
1955 int len;
1956
1957 if (xen_has_pv_and_legacy_disk_devices()) {
1958 int major;
1959
1960 if (!VDEV_IS_EXTENDED(vdevice))
1961 major = BLKIF_MAJOR(vdevice);
1962 else
1963 major = XENVBD_MAJOR;
1964
1965 if (major != XENVBD_MAJOR) {
1966 printk(KERN_INFO
1967 "%s: HVM does not support vbd %d as xen block device\n",
1968 __func__, vdevice);
1969 return -ENODEV;
1970 }
1971 }
1972
1973 type = xenbus_read(XBT_NIL, dev->nodename, "device-type", &len);
1974 if (IS_ERR(type))
1975 return -ENODEV;
1976 if (strncmp(type, "cdrom", 5) == 0) {
1977 kfree(type);
1978 return -ENODEV;
1979 }
1980 kfree(type);
1981 }
1982 info = kzalloc(sizeof(*info), GFP_KERNEL);
1983 if (!info) {
1984 xenbus_dev_fatal(dev, -ENOMEM, "allocating info structure");
1985 return -ENOMEM;
1986 }
1987
1988 info->xbdev = dev;
1989
1990 mutex_init(&info->mutex);
1991 info->vdevice = vdevice;
1992 info->connected = BLKIF_STATE_DISCONNECTED;
1993
1994
1995 info->handle = simple_strtoul(strrchr(dev->nodename, '/')+1, NULL, 0);
1996 dev_set_drvdata(&dev->dev, info);
1997
1998 return 0;
1999}
2000
2001static int blkif_recover(struct blkfront_info *info)
2002{
2003 unsigned int r_index;
2004 struct request *req, *n;
2005 int rc;
2006 struct bio *bio;
2007 unsigned int segs;
2008
2009 blkfront_gather_backend_features(info);
2010
2011 blkif_set_queue_limits(info);
2012 segs = info->max_indirect_segments ? : BLKIF_MAX_SEGMENTS_PER_REQUEST;
2013 blk_queue_max_segments(info->rq, segs / GRANTS_PER_PSEG);
2014
2015 for (r_index = 0; r_index < info->nr_rings; r_index++) {
2016 struct blkfront_ring_info *rinfo = &info->rinfo[r_index];
2017
2018 rc = blkfront_setup_indirect(rinfo);
2019 if (rc)
2020 return rc;
2021 }
2022 xenbus_switch_state(info->xbdev, XenbusStateConnected);
2023
2024
2025 info->connected = BLKIF_STATE_CONNECTED;
2026
2027 for (r_index = 0; r_index < info->nr_rings; r_index++) {
2028 struct blkfront_ring_info *rinfo;
2029
2030 rinfo = &info->rinfo[r_index];
2031
2032 kick_pending_request_queues(rinfo);
2033 }
2034
2035 list_for_each_entry_safe(req, n, &info->requests, queuelist) {
2036
2037 list_del_init(&req->queuelist);
2038 BUG_ON(req->nr_phys_segments > segs);
2039 blk_mq_requeue_request(req, false);
2040 }
2041 blk_mq_start_stopped_hw_queues(info->rq, true);
2042 blk_mq_kick_requeue_list(info->rq);
2043
2044 while ((bio = bio_list_pop(&info->bio_list)) != NULL) {
2045
2046 submit_bio(bio);
2047 }
2048
2049 return 0;
2050}
2051
2052
2053
2054
2055
2056
2057
2058static int blkfront_resume(struct xenbus_device *dev)
2059{
2060 struct blkfront_info *info = dev_get_drvdata(&dev->dev);
2061 int err = 0;
2062 unsigned int i, j;
2063
2064 dev_dbg(&dev->dev, "blkfront_resume: %s\n", dev->nodename);
2065
2066 bio_list_init(&info->bio_list);
2067 INIT_LIST_HEAD(&info->requests);
2068 for (i = 0; i < info->nr_rings; i++) {
2069 struct blkfront_ring_info *rinfo = &info->rinfo[i];
2070 struct bio_list merge_bio;
2071 struct blk_shadow *shadow = rinfo->shadow;
2072
2073 for (j = 0; j < BLK_RING_SIZE(info); j++) {
2074
2075 if (!shadow[j].request)
2076 continue;
2077
2078
2079
2080
2081 if (req_op(shadow[j].request) == REQ_OP_FLUSH ||
2082 req_op(shadow[j].request) == REQ_OP_DISCARD ||
2083 req_op(shadow[j].request) == REQ_OP_SECURE_ERASE ||
2084 shadow[j].request->cmd_flags & REQ_FUA) {
2085
2086
2087
2088
2089
2090
2091
2092 list_add(&shadow[j].request->queuelist, &info->requests);
2093 continue;
2094 }
2095 merge_bio.head = shadow[j].request->bio;
2096 merge_bio.tail = shadow[j].request->biotail;
2097 bio_list_merge(&info->bio_list, &merge_bio);
2098 shadow[j].request->bio = NULL;
2099 blk_mq_end_request(shadow[j].request, BLK_STS_OK);
2100 }
2101 }
2102
2103 blkif_free(info, info->connected == BLKIF_STATE_CONNECTED);
2104
2105 err = talk_to_blkback(dev, info);
2106 if (!err)
2107 blk_mq_update_nr_hw_queues(&info->tag_set, info->nr_rings);
2108
2109
2110
2111
2112
2113
2114
2115 return err;
2116}
2117
2118static void blkfront_closing(struct blkfront_info *info)
2119{
2120 struct xenbus_device *xbdev = info->xbdev;
2121 struct block_device *bdev = NULL;
2122
2123 mutex_lock(&info->mutex);
2124
2125 if (xbdev->state == XenbusStateClosing) {
2126 mutex_unlock(&info->mutex);
2127 return;
2128 }
2129
2130 if (info->gd)
2131 bdev = bdget_disk(info->gd, 0);
2132
2133 mutex_unlock(&info->mutex);
2134
2135 if (!bdev) {
2136 xenbus_frontend_closed(xbdev);
2137 return;
2138 }
2139
2140 mutex_lock(&bdev->bd_mutex);
2141
2142 if (bdev->bd_openers) {
2143 xenbus_dev_error(xbdev, -EBUSY,
2144 "Device in use; refusing to close");
2145 xenbus_switch_state(xbdev, XenbusStateClosing);
2146 } else {
2147 xlvbd_release_gendisk(info);
2148 xenbus_frontend_closed(xbdev);
2149 }
2150
2151 mutex_unlock(&bdev->bd_mutex);
2152 bdput(bdev);
2153}
2154
2155static void blkfront_setup_discard(struct blkfront_info *info)
2156{
2157 int err;
2158 unsigned int discard_granularity;
2159 unsigned int discard_alignment;
2160
2161 info->feature_discard = 1;
2162 err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
2163 "discard-granularity", "%u", &discard_granularity,
2164 "discard-alignment", "%u", &discard_alignment,
2165 NULL);
2166 if (!err) {
2167 info->discard_granularity = discard_granularity;
2168 info->discard_alignment = discard_alignment;
2169 }
2170 info->feature_secdiscard =
2171 !!xenbus_read_unsigned(info->xbdev->otherend, "discard-secure",
2172 0);
2173}
2174
2175static int blkfront_setup_indirect(struct blkfront_ring_info *rinfo)
2176{
2177 unsigned int psegs, grants;
2178 int err, i;
2179 struct blkfront_info *info = rinfo->dev_info;
2180
2181 if (info->max_indirect_segments == 0) {
2182 if (!HAS_EXTRA_REQ)
2183 grants = BLKIF_MAX_SEGMENTS_PER_REQUEST;
2184 else {
2185
2186
2187
2188
2189
2190 grants = GRANTS_PER_PSEG;
2191 }
2192 }
2193 else
2194 grants = info->max_indirect_segments;
2195 psegs = DIV_ROUND_UP(grants, GRANTS_PER_PSEG);
2196
2197 err = fill_grant_buffer(rinfo,
2198 (grants + INDIRECT_GREFS(grants)) * BLK_RING_SIZE(info));
2199 if (err)
2200 goto out_of_memory;
2201
2202 if (!info->feature_persistent && info->max_indirect_segments) {
2203
2204
2205
2206
2207
2208 int num = INDIRECT_GREFS(grants) * BLK_RING_SIZE(info);
2209
2210 BUG_ON(!list_empty(&rinfo->indirect_pages));
2211 for (i = 0; i < num; i++) {
2212 struct page *indirect_page = alloc_page(GFP_NOIO);
2213 if (!indirect_page)
2214 goto out_of_memory;
2215 list_add(&indirect_page->lru, &rinfo->indirect_pages);
2216 }
2217 }
2218
2219 for (i = 0; i < BLK_RING_SIZE(info); i++) {
2220 rinfo->shadow[i].grants_used = kzalloc(
2221 sizeof(rinfo->shadow[i].grants_used[0]) * grants,
2222 GFP_NOIO);
2223 rinfo->shadow[i].sg = kzalloc(sizeof(rinfo->shadow[i].sg[0]) * psegs, GFP_NOIO);
2224 if (info->max_indirect_segments)
2225 rinfo->shadow[i].indirect_grants = kzalloc(
2226 sizeof(rinfo->shadow[i].indirect_grants[0]) *
2227 INDIRECT_GREFS(grants),
2228 GFP_NOIO);
2229 if ((rinfo->shadow[i].grants_used == NULL) ||
2230 (rinfo->shadow[i].sg == NULL) ||
2231 (info->max_indirect_segments &&
2232 (rinfo->shadow[i].indirect_grants == NULL)))
2233 goto out_of_memory;
2234 sg_init_table(rinfo->shadow[i].sg, psegs);
2235 }
2236
2237
2238 return 0;
2239
2240out_of_memory:
2241 for (i = 0; i < BLK_RING_SIZE(info); i++) {
2242 kfree(rinfo->shadow[i].grants_used);
2243 rinfo->shadow[i].grants_used = NULL;
2244 kfree(rinfo->shadow[i].sg);
2245 rinfo->shadow[i].sg = NULL;
2246 kfree(rinfo->shadow[i].indirect_grants);
2247 rinfo->shadow[i].indirect_grants = NULL;
2248 }
2249 if (!list_empty(&rinfo->indirect_pages)) {
2250 struct page *indirect_page, *n;
2251 list_for_each_entry_safe(indirect_page, n, &rinfo->indirect_pages, lru) {
2252 list_del(&indirect_page->lru);
2253 __free_page(indirect_page);
2254 }
2255 }
2256 return -ENOMEM;
2257}
2258
2259
2260
2261
2262static void blkfront_gather_backend_features(struct blkfront_info *info)
2263{
2264 unsigned int indirect_segments;
2265
2266 info->feature_flush = 0;
2267 info->feature_fua = 0;
2268
2269
2270
2271
2272
2273
2274
2275
2276 if (xenbus_read_unsigned(info->xbdev->otherend, "feature-barrier", 0)) {
2277 info->feature_flush = 1;
2278 info->feature_fua = 1;
2279 }
2280
2281
2282
2283
2284
2285 if (xenbus_read_unsigned(info->xbdev->otherend, "feature-flush-cache",
2286 0)) {
2287 info->feature_flush = 1;
2288 info->feature_fua = 0;
2289 }
2290
2291 if (xenbus_read_unsigned(info->xbdev->otherend, "feature-discard", 0))
2292 blkfront_setup_discard(info);
2293
2294 info->feature_persistent =
2295 !!xenbus_read_unsigned(info->xbdev->otherend,
2296 "feature-persistent", 0);
2297
2298 indirect_segments = xenbus_read_unsigned(info->xbdev->otherend,
2299 "feature-max-indirect-segments", 0);
2300 if (indirect_segments > xen_blkif_max_segments)
2301 indirect_segments = xen_blkif_max_segments;
2302 if (indirect_segments <= BLKIF_MAX_SEGMENTS_PER_REQUEST)
2303 indirect_segments = 0;
2304 info->max_indirect_segments = indirect_segments;
2305}
2306
2307
2308
2309
2310
2311static void blkfront_connect(struct blkfront_info *info)
2312{
2313 unsigned long long sectors;
2314 unsigned long sector_size;
2315 unsigned int physical_sector_size;
2316 unsigned int binfo;
2317 char *envp[] = { "RESIZE=1", NULL };
2318 int err, i;
2319
2320 switch (info->connected) {
2321 case BLKIF_STATE_CONNECTED:
2322
2323
2324
2325
2326 err = xenbus_scanf(XBT_NIL, info->xbdev->otherend,
2327 "sectors", "%Lu", §ors);
2328 if (XENBUS_EXIST_ERR(err))
2329 return;
2330 printk(KERN_INFO "Setting capacity to %Lu\n",
2331 sectors);
2332 set_capacity(info->gd, sectors);
2333 revalidate_disk(info->gd);
2334 kobject_uevent_env(&disk_to_dev(info->gd)->kobj,
2335 KOBJ_CHANGE, envp);
2336
2337 return;
2338 case BLKIF_STATE_SUSPENDED:
2339
2340
2341
2342
2343
2344
2345 blkif_recover(info);
2346 return;
2347
2348 default:
2349 break;
2350 }
2351
2352 dev_dbg(&info->xbdev->dev, "%s:%s.\n",
2353 __func__, info->xbdev->otherend);
2354
2355 err = xenbus_gather(XBT_NIL, info->xbdev->otherend,
2356 "sectors", "%llu", §ors,
2357 "info", "%u", &binfo,
2358 "sector-size", "%lu", §or_size,
2359 NULL);
2360 if (err) {
2361 xenbus_dev_fatal(info->xbdev, err,
2362 "reading backend fields at %s",
2363 info->xbdev->otherend);
2364 return;
2365 }
2366
2367
2368
2369
2370
2371
2372 physical_sector_size = xenbus_read_unsigned(info->xbdev->otherend,
2373 "physical-sector-size",
2374 sector_size);
2375 blkfront_gather_backend_features(info);
2376 for (i = 0; i < info->nr_rings; i++) {
2377 err = blkfront_setup_indirect(&info->rinfo[i]);
2378 if (err) {
2379 xenbus_dev_fatal(info->xbdev, err, "setup_indirect at %s",
2380 info->xbdev->otherend);
2381 blkif_free(info, 0);
2382 break;
2383 }
2384 }
2385
2386 err = xlvbd_alloc_gendisk(sectors, info, binfo, sector_size,
2387 physical_sector_size);
2388 if (err) {
2389 xenbus_dev_fatal(info->xbdev, err, "xlvbd_add at %s",
2390 info->xbdev->otherend);
2391 goto fail;
2392 }
2393
2394 xenbus_switch_state(info->xbdev, XenbusStateConnected);
2395
2396
2397 info->connected = BLKIF_STATE_CONNECTED;
2398 for (i = 0; i < info->nr_rings; i++)
2399 kick_pending_request_queues(&info->rinfo[i]);
2400
2401 device_add_disk(&info->xbdev->dev, info->gd);
2402
2403 info->is_ready = 1;
2404 return;
2405
2406fail:
2407 blkif_free(info, 0);
2408 return;
2409}
2410
2411
2412
2413
2414static void blkback_changed(struct xenbus_device *dev,
2415 enum xenbus_state backend_state)
2416{
2417 struct blkfront_info *info = dev_get_drvdata(&dev->dev);
2418
2419 dev_dbg(&dev->dev, "blkfront:blkback_changed to state %d.\n", backend_state);
2420
2421 switch (backend_state) {
2422 case XenbusStateInitWait:
2423 if (dev->state != XenbusStateInitialising)
2424 break;
2425 if (talk_to_blkback(dev, info))
2426 break;
2427 case XenbusStateInitialising:
2428 case XenbusStateInitialised:
2429 case XenbusStateReconfiguring:
2430 case XenbusStateReconfigured:
2431 case XenbusStateUnknown:
2432 break;
2433
2434 case XenbusStateConnected:
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446 if ((dev->state != XenbusStateInitialised) &&
2447 (dev->state != XenbusStateConnected)) {
2448 if (talk_to_blkback(dev, info))
2449 break;
2450 }
2451
2452 blkfront_connect(info);
2453 break;
2454
2455 case XenbusStateClosed:
2456 if (dev->state == XenbusStateClosed)
2457 break;
2458
2459 case XenbusStateClosing:
2460 if (info)
2461 blkfront_closing(info);
2462 break;
2463 }
2464}
2465
2466static int blkfront_remove(struct xenbus_device *xbdev)
2467{
2468 struct blkfront_info *info = dev_get_drvdata(&xbdev->dev);
2469 struct block_device *bdev = NULL;
2470 struct gendisk *disk;
2471
2472 dev_dbg(&xbdev->dev, "%s removed", xbdev->nodename);
2473
2474 blkif_free(info, 0);
2475
2476 mutex_lock(&info->mutex);
2477
2478 disk = info->gd;
2479 if (disk)
2480 bdev = bdget_disk(disk, 0);
2481
2482 info->xbdev = NULL;
2483 mutex_unlock(&info->mutex);
2484
2485 if (!bdev) {
2486 kfree(info);
2487 return 0;
2488 }
2489
2490
2491
2492
2493
2494
2495
2496 mutex_lock(&bdev->bd_mutex);
2497 info = disk->private_data;
2498
2499 dev_warn(disk_to_dev(disk),
2500 "%s was hot-unplugged, %d stale handles\n",
2501 xbdev->nodename, bdev->bd_openers);
2502
2503 if (info && !bdev->bd_openers) {
2504 xlvbd_release_gendisk(info);
2505 disk->private_data = NULL;
2506 kfree(info);
2507 }
2508
2509 mutex_unlock(&bdev->bd_mutex);
2510 bdput(bdev);
2511
2512 return 0;
2513}
2514
2515static int blkfront_is_ready(struct xenbus_device *dev)
2516{
2517 struct blkfront_info *info = dev_get_drvdata(&dev->dev);
2518
2519 return info->is_ready && info->xbdev;
2520}
2521
2522static int blkif_open(struct block_device *bdev, fmode_t mode)
2523{
2524 struct gendisk *disk = bdev->bd_disk;
2525 struct blkfront_info *info;
2526 int err = 0;
2527
2528 mutex_lock(&blkfront_mutex);
2529
2530 info = disk->private_data;
2531 if (!info) {
2532
2533 err = -ERESTARTSYS;
2534 goto out;
2535 }
2536
2537 mutex_lock(&info->mutex);
2538
2539 if (!info->gd)
2540
2541 err = -ERESTARTSYS;
2542
2543 mutex_unlock(&info->mutex);
2544
2545out:
2546 mutex_unlock(&blkfront_mutex);
2547 return err;
2548}
2549
2550static void blkif_release(struct gendisk *disk, fmode_t mode)
2551{
2552 struct blkfront_info *info = disk->private_data;
2553 struct block_device *bdev;
2554 struct xenbus_device *xbdev;
2555
2556 mutex_lock(&blkfront_mutex);
2557
2558 bdev = bdget_disk(disk, 0);
2559
2560 if (!bdev) {
2561 WARN(1, "Block device %s yanked out from us!\n", disk->disk_name);
2562 goto out_mutex;
2563 }
2564 if (bdev->bd_openers)
2565 goto out;
2566
2567
2568
2569
2570
2571
2572 mutex_lock(&info->mutex);
2573 xbdev = info->xbdev;
2574
2575 if (xbdev && xbdev->state == XenbusStateClosing) {
2576
2577 dev_info(disk_to_dev(bdev->bd_disk), "releasing disk\n");
2578 xlvbd_release_gendisk(info);
2579 xenbus_frontend_closed(info->xbdev);
2580 }
2581
2582 mutex_unlock(&info->mutex);
2583
2584 if (!xbdev) {
2585
2586 dev_info(disk_to_dev(bdev->bd_disk), "releasing disk\n");
2587 xlvbd_release_gendisk(info);
2588 disk->private_data = NULL;
2589 kfree(info);
2590 }
2591
2592out:
2593 bdput(bdev);
2594out_mutex:
2595 mutex_unlock(&blkfront_mutex);
2596}
2597
2598static const struct block_device_operations xlvbd_block_fops =
2599{
2600 .owner = THIS_MODULE,
2601 .open = blkif_open,
2602 .release = blkif_release,
2603 .getgeo = blkif_getgeo,
2604 .ioctl = blkif_ioctl,
2605};
2606
2607
2608static const struct xenbus_device_id blkfront_ids[] = {
2609 { "vbd" },
2610 { "" }
2611};
2612
2613static struct xenbus_driver blkfront_driver = {
2614 .ids = blkfront_ids,
2615 .probe = blkfront_probe,
2616 .remove = blkfront_remove,
2617 .resume = blkfront_resume,
2618 .otherend_changed = blkback_changed,
2619 .is_ready = blkfront_is_ready,
2620};
2621
2622static int __init xlblk_init(void)
2623{
2624 int ret;
2625 int nr_cpus = num_online_cpus();
2626
2627 if (!xen_domain())
2628 return -ENODEV;
2629
2630 if (xen_blkif_max_segments < BLKIF_MAX_SEGMENTS_PER_REQUEST)
2631 xen_blkif_max_segments = BLKIF_MAX_SEGMENTS_PER_REQUEST;
2632
2633 if (xen_blkif_max_ring_order > XENBUS_MAX_RING_GRANT_ORDER) {
2634 pr_info("Invalid max_ring_order (%d), will use default max: %d.\n",
2635 xen_blkif_max_ring_order, XENBUS_MAX_RING_GRANT_ORDER);
2636 xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
2637 }
2638
2639 if (xen_blkif_max_queues > nr_cpus) {
2640 pr_info("Invalid max_queues (%d), will use default max: %d.\n",
2641 xen_blkif_max_queues, nr_cpus);
2642 xen_blkif_max_queues = nr_cpus;
2643 }
2644
2645 if (!xen_has_pv_disk_devices())
2646 return -ENODEV;
2647
2648 if (register_blkdev(XENVBD_MAJOR, DEV_NAME)) {
2649 printk(KERN_WARNING "xen_blk: can't get major %d with name %s\n",
2650 XENVBD_MAJOR, DEV_NAME);
2651 return -ENODEV;
2652 }
2653
2654 ret = xenbus_register_frontend(&blkfront_driver);
2655 if (ret) {
2656 unregister_blkdev(XENVBD_MAJOR, DEV_NAME);
2657 return ret;
2658 }
2659
2660 return 0;
2661}
2662module_init(xlblk_init);
2663
2664
2665static void __exit xlblk_exit(void)
2666{
2667 xenbus_unregister_driver(&blkfront_driver);
2668 unregister_blkdev(XENVBD_MAJOR, DEV_NAME);
2669 kfree(minors);
2670}
2671module_exit(xlblk_exit);
2672
2673MODULE_DESCRIPTION("Xen virtual block device frontend");
2674MODULE_LICENSE("GPL");
2675MODULE_ALIAS_BLOCKDEV_MAJOR(XENVBD_MAJOR);
2676MODULE_ALIAS("xen:vbd");
2677MODULE_ALIAS("xenblk");
2678