1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/mm.h>
34#include <linux/slab.h>
35#include <linux/types.h>
36#include <linux/spinlock.h>
37#include <linux/vmalloc.h>
38#include <linux/export.h>
39#include <asm/xen/hypervisor.h>
40#include <xen/page.h>
41#include <xen/interface/xen.h>
42#include <xen/interface/event_channel.h>
43#include <xen/balloon.h>
44#include <xen/events.h>
45#include <xen/grant_table.h>
46#include <xen/xenbus.h>
47#include <xen/xen.h>
48#include <xen/features.h>
49
50#include "xenbus.h"
51
52#define XENBUS_PAGES(_grants) (DIV_ROUND_UP(_grants, XEN_PFN_PER_PAGE))
53
54#define XENBUS_MAX_RING_PAGES (XENBUS_PAGES(XENBUS_MAX_RING_GRANTS))
55
56struct xenbus_map_node {
57 struct list_head next;
58 union {
59 struct {
60 struct vm_struct *area;
61 } pv;
62 struct {
63 struct page *pages[XENBUS_MAX_RING_PAGES];
64 unsigned long addrs[XENBUS_MAX_RING_GRANTS];
65 void *addr;
66 } hvm;
67 };
68 grant_handle_t handles[XENBUS_MAX_RING_GRANTS];
69 unsigned int nr_handles;
70};
71
72struct map_ring_valloc {
73 struct xenbus_map_node *node;
74
75
76 union {
77 unsigned long addrs[XENBUS_MAX_RING_GRANTS];
78 pte_t *ptes[XENBUS_MAX_RING_GRANTS];
79 };
80 phys_addr_t phys_addrs[XENBUS_MAX_RING_GRANTS];
81
82 struct gnttab_map_grant_ref map[XENBUS_MAX_RING_GRANTS];
83 struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_GRANTS];
84
85 unsigned int idx;
86};
87
88static DEFINE_SPINLOCK(xenbus_valloc_lock);
89static LIST_HEAD(xenbus_valloc_pages);
90
91struct xenbus_ring_ops {
92 int (*map)(struct xenbus_device *dev, struct map_ring_valloc *info,
93 grant_ref_t *gnt_refs, unsigned int nr_grefs,
94 void **vaddr);
95 int (*unmap)(struct xenbus_device *dev, void *vaddr);
96};
97
98static const struct xenbus_ring_ops *ring_ops __read_mostly;
99
100const char *xenbus_strstate(enum xenbus_state state)
101{
102 static const char *const name[] = {
103 [ XenbusStateUnknown ] = "Unknown",
104 [ XenbusStateInitialising ] = "Initialising",
105 [ XenbusStateInitWait ] = "InitWait",
106 [ XenbusStateInitialised ] = "Initialised",
107 [ XenbusStateConnected ] = "Connected",
108 [ XenbusStateClosing ] = "Closing",
109 [ XenbusStateClosed ] = "Closed",
110 [XenbusStateReconfiguring] = "Reconfiguring",
111 [XenbusStateReconfigured] = "Reconfigured",
112 };
113 return (state < ARRAY_SIZE(name)) ? name[state] : "INVALID";
114}
115EXPORT_SYMBOL_GPL(xenbus_strstate);
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131int xenbus_watch_path(struct xenbus_device *dev, const char *path,
132 struct xenbus_watch *watch,
133 void (*callback)(struct xenbus_watch *,
134 const char *, const char *))
135{
136 int err;
137
138 watch->node = path;
139 watch->callback = callback;
140
141 err = register_xenbus_watch(watch);
142
143 if (err) {
144 watch->node = NULL;
145 watch->callback = NULL;
146 xenbus_dev_fatal(dev, err, "adding watch on %s", path);
147 }
148
149 return err;
150}
151EXPORT_SYMBOL_GPL(xenbus_watch_path);
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169int xenbus_watch_pathfmt(struct xenbus_device *dev,
170 struct xenbus_watch *watch,
171 void (*callback)(struct xenbus_watch *,
172 const char *, const char *),
173 const char *pathfmt, ...)
174{
175 int err;
176 va_list ap;
177 char *path;
178
179 va_start(ap, pathfmt);
180 path = kvasprintf(GFP_NOIO | __GFP_HIGH, pathfmt, ap);
181 va_end(ap);
182
183 if (!path) {
184 xenbus_dev_fatal(dev, -ENOMEM, "allocating path for watch");
185 return -ENOMEM;
186 }
187 err = xenbus_watch_path(dev, path, watch, callback);
188
189 if (err)
190 kfree(path);
191 return err;
192}
193EXPORT_SYMBOL_GPL(xenbus_watch_pathfmt);
194
195static void xenbus_switch_fatal(struct xenbus_device *, int, int,
196 const char *, ...);
197
198static int
199__xenbus_switch_state(struct xenbus_device *dev,
200 enum xenbus_state state, int depth)
201{
202
203
204
205
206
207
208
209
210
211
212
213
214
215 struct xenbus_transaction xbt;
216 int current_state;
217 int err, abort;
218
219 if (state == dev->state)
220 return 0;
221
222again:
223 abort = 1;
224
225 err = xenbus_transaction_start(&xbt);
226 if (err) {
227 xenbus_switch_fatal(dev, depth, err, "starting transaction");
228 return 0;
229 }
230
231 err = xenbus_scanf(xbt, dev->nodename, "state", "%d", ¤t_state);
232 if (err != 1)
233 goto abort;
234
235 err = xenbus_printf(xbt, dev->nodename, "state", "%d", state);
236 if (err) {
237 xenbus_switch_fatal(dev, depth, err, "writing new state");
238 goto abort;
239 }
240
241 abort = 0;
242abort:
243 err = xenbus_transaction_end(xbt, abort);
244 if (err) {
245 if (err == -EAGAIN && !abort)
246 goto again;
247 xenbus_switch_fatal(dev, depth, err, "ending transaction");
248 } else
249 dev->state = state;
250
251 return 0;
252}
253
254
255
256
257
258
259
260
261
262
263int xenbus_switch_state(struct xenbus_device *dev, enum xenbus_state state)
264{
265 return __xenbus_switch_state(dev, state, 0);
266}
267
268EXPORT_SYMBOL_GPL(xenbus_switch_state);
269
270int xenbus_frontend_closed(struct xenbus_device *dev)
271{
272 xenbus_switch_state(dev, XenbusStateClosed);
273 complete(&dev->down);
274 return 0;
275}
276EXPORT_SYMBOL_GPL(xenbus_frontend_closed);
277
278static void xenbus_va_dev_error(struct xenbus_device *dev, int err,
279 const char *fmt, va_list ap)
280{
281 unsigned int len;
282 char *printf_buffer;
283 char *path_buffer;
284
285#define PRINTF_BUFFER_SIZE 4096
286
287 printf_buffer = kmalloc(PRINTF_BUFFER_SIZE, GFP_KERNEL);
288 if (!printf_buffer)
289 return;
290
291 len = sprintf(printf_buffer, "%i ", -err);
292 vsnprintf(printf_buffer + len, PRINTF_BUFFER_SIZE - len, fmt, ap);
293
294 dev_err(&dev->dev, "%s\n", printf_buffer);
295
296 path_buffer = kasprintf(GFP_KERNEL, "error/%s", dev->nodename);
297 if (path_buffer)
298 xenbus_write(XBT_NIL, path_buffer, "error", printf_buffer);
299
300 kfree(printf_buffer);
301 kfree(path_buffer);
302}
303
304
305
306
307
308
309
310
311
312
313void xenbus_dev_error(struct xenbus_device *dev, int err, const char *fmt, ...)
314{
315 va_list ap;
316
317 va_start(ap, fmt);
318 xenbus_va_dev_error(dev, err, fmt, ap);
319 va_end(ap);
320}
321EXPORT_SYMBOL_GPL(xenbus_dev_error);
322
323
324
325
326
327
328
329
330
331
332
333
334void xenbus_dev_fatal(struct xenbus_device *dev, int err, const char *fmt, ...)
335{
336 va_list ap;
337
338 va_start(ap, fmt);
339 xenbus_va_dev_error(dev, err, fmt, ap);
340 va_end(ap);
341
342 xenbus_switch_state(dev, XenbusStateClosing);
343}
344EXPORT_SYMBOL_GPL(xenbus_dev_fatal);
345
346
347
348
349
350static void xenbus_switch_fatal(struct xenbus_device *dev, int depth, int err,
351 const char *fmt, ...)
352{
353 va_list ap;
354
355 va_start(ap, fmt);
356 xenbus_va_dev_error(dev, err, fmt, ap);
357 va_end(ap);
358
359 if (!depth)
360 __xenbus_switch_state(dev, XenbusStateClosing, 1);
361}
362
363
364
365
366
367
368
369
370
371
372
373
374
375int xenbus_grant_ring(struct xenbus_device *dev, void *vaddr,
376 unsigned int nr_pages, grant_ref_t *grefs)
377{
378 int err;
379 int i, j;
380
381 for (i = 0; i < nr_pages; i++) {
382 unsigned long gfn;
383
384 if (is_vmalloc_addr(vaddr))
385 gfn = pfn_to_gfn(vmalloc_to_pfn(vaddr));
386 else
387 gfn = virt_to_gfn(vaddr);
388
389 err = gnttab_grant_foreign_access(dev->otherend_id, gfn, 0);
390 if (err < 0) {
391 xenbus_dev_fatal(dev, err,
392 "granting access to ring page");
393 goto fail;
394 }
395 grefs[i] = err;
396
397 vaddr = vaddr + XEN_PAGE_SIZE;
398 }
399
400 return 0;
401
402fail:
403 for (j = 0; j < i; j++)
404 gnttab_end_foreign_access_ref(grefs[j], 0);
405 return err;
406}
407EXPORT_SYMBOL_GPL(xenbus_grant_ring);
408
409
410
411
412
413
414
415
416int xenbus_alloc_evtchn(struct xenbus_device *dev, evtchn_port_t *port)
417{
418 struct evtchn_alloc_unbound alloc_unbound;
419 int err;
420
421 alloc_unbound.dom = DOMID_SELF;
422 alloc_unbound.remote_dom = dev->otherend_id;
423
424 err = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
425 &alloc_unbound);
426 if (err)
427 xenbus_dev_fatal(dev, err, "allocating event channel");
428 else
429 *port = alloc_unbound.port;
430
431 return err;
432}
433EXPORT_SYMBOL_GPL(xenbus_alloc_evtchn);
434
435
436
437
438
439int xenbus_free_evtchn(struct xenbus_device *dev, evtchn_port_t port)
440{
441 struct evtchn_close close;
442 int err;
443
444 close.port = port;
445
446 err = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
447 if (err)
448 xenbus_dev_error(dev, err, "freeing event channel %u", port);
449
450 return err;
451}
452EXPORT_SYMBOL_GPL(xenbus_free_evtchn);
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469int xenbus_map_ring_valloc(struct xenbus_device *dev, grant_ref_t *gnt_refs,
470 unsigned int nr_grefs, void **vaddr)
471{
472 int err;
473 struct map_ring_valloc *info;
474
475 *vaddr = NULL;
476
477 if (nr_grefs > XENBUS_MAX_RING_GRANTS)
478 return -EINVAL;
479
480 info = kzalloc(sizeof(*info), GFP_KERNEL);
481 if (!info)
482 return -ENOMEM;
483
484 info->node = kzalloc(sizeof(*info->node), GFP_KERNEL);
485 if (!info->node)
486 err = -ENOMEM;
487 else
488 err = ring_ops->map(dev, info, gnt_refs, nr_grefs, vaddr);
489
490 kfree(info->node);
491 kfree(info);
492 return err;
493}
494EXPORT_SYMBOL_GPL(xenbus_map_ring_valloc);
495
496
497
498
499static int __xenbus_map_ring(struct xenbus_device *dev,
500 grant_ref_t *gnt_refs,
501 unsigned int nr_grefs,
502 grant_handle_t *handles,
503 struct map_ring_valloc *info,
504 unsigned int flags,
505 bool *leaked)
506{
507 int i, j;
508
509 if (nr_grefs > XENBUS_MAX_RING_GRANTS)
510 return -EINVAL;
511
512 for (i = 0; i < nr_grefs; i++) {
513 gnttab_set_map_op(&info->map[i], info->phys_addrs[i], flags,
514 gnt_refs[i], dev->otherend_id);
515 handles[i] = INVALID_GRANT_HANDLE;
516 }
517
518 gnttab_batch_map(info->map, i);
519
520 for (i = 0; i < nr_grefs; i++) {
521 if (info->map[i].status != GNTST_okay) {
522 xenbus_dev_fatal(dev, info->map[i].status,
523 "mapping in shared page %d from domain %d",
524 gnt_refs[i], dev->otherend_id);
525 goto fail;
526 } else
527 handles[i] = info->map[i].handle;
528 }
529
530 return 0;
531
532 fail:
533 for (i = j = 0; i < nr_grefs; i++) {
534 if (handles[i] != INVALID_GRANT_HANDLE) {
535 gnttab_set_unmap_op(&info->unmap[j],
536 info->phys_addrs[i],
537 GNTMAP_host_map, handles[i]);
538 j++;
539 }
540 }
541
542 if (HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, info->unmap, j))
543 BUG();
544
545 *leaked = false;
546 for (i = 0; i < j; i++) {
547 if (info->unmap[i].status != GNTST_okay) {
548 *leaked = true;
549 break;
550 }
551 }
552
553 return -ENOENT;
554}
555
556
557
558
559
560
561
562
563
564
565
566
567static int xenbus_unmap_ring(struct xenbus_device *dev, grant_handle_t *handles,
568 unsigned int nr_handles, unsigned long *vaddrs)
569{
570 struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_GRANTS];
571 int i;
572 int err;
573
574 if (nr_handles > XENBUS_MAX_RING_GRANTS)
575 return -EINVAL;
576
577 for (i = 0; i < nr_handles; i++)
578 gnttab_set_unmap_op(&unmap[i], vaddrs[i],
579 GNTMAP_host_map, handles[i]);
580
581 if (HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap, i))
582 BUG();
583
584 err = GNTST_okay;
585 for (i = 0; i < nr_handles; i++) {
586 if (unmap[i].status != GNTST_okay) {
587 xenbus_dev_error(dev, unmap[i].status,
588 "unmapping page at handle %d error %d",
589 handles[i], unmap[i].status);
590 err = unmap[i].status;
591 break;
592 }
593 }
594
595 return err;
596}
597
598static void xenbus_map_ring_setup_grant_hvm(unsigned long gfn,
599 unsigned int goffset,
600 unsigned int len,
601 void *data)
602{
603 struct map_ring_valloc *info = data;
604 unsigned long vaddr = (unsigned long)gfn_to_virt(gfn);
605
606 info->phys_addrs[info->idx] = vaddr;
607 info->addrs[info->idx] = vaddr;
608
609 info->idx++;
610}
611
612static int xenbus_map_ring_hvm(struct xenbus_device *dev,
613 struct map_ring_valloc *info,
614 grant_ref_t *gnt_ref,
615 unsigned int nr_grefs,
616 void **vaddr)
617{
618 struct xenbus_map_node *node = info->node;
619 int err;
620 void *addr;
621 bool leaked = false;
622 unsigned int nr_pages = XENBUS_PAGES(nr_grefs);
623
624 err = xen_alloc_unpopulated_pages(nr_pages, node->hvm.pages);
625 if (err)
626 goto out_err;
627
628 gnttab_foreach_grant(node->hvm.pages, nr_grefs,
629 xenbus_map_ring_setup_grant_hvm,
630 info);
631
632 err = __xenbus_map_ring(dev, gnt_ref, nr_grefs, node->handles,
633 info, GNTMAP_host_map, &leaked);
634 node->nr_handles = nr_grefs;
635
636 if (err)
637 goto out_free_ballooned_pages;
638
639 addr = vmap(node->hvm.pages, nr_pages, VM_MAP | VM_IOREMAP,
640 PAGE_KERNEL);
641 if (!addr) {
642 err = -ENOMEM;
643 goto out_xenbus_unmap_ring;
644 }
645
646 node->hvm.addr = addr;
647
648 spin_lock(&xenbus_valloc_lock);
649 list_add(&node->next, &xenbus_valloc_pages);
650 spin_unlock(&xenbus_valloc_lock);
651
652 *vaddr = addr;
653 info->node = NULL;
654
655 return 0;
656
657 out_xenbus_unmap_ring:
658 if (!leaked)
659 xenbus_unmap_ring(dev, node->handles, nr_grefs, info->addrs);
660 else
661 pr_alert("leaking %p size %u page(s)",
662 addr, nr_pages);
663 out_free_ballooned_pages:
664 if (!leaked)
665 xen_free_unpopulated_pages(nr_pages, node->hvm.pages);
666 out_err:
667 return err;
668}
669
670
671
672
673
674
675
676
677
678
679
680
681
682int xenbus_unmap_ring_vfree(struct xenbus_device *dev, void *vaddr)
683{
684 return ring_ops->unmap(dev, vaddr);
685}
686EXPORT_SYMBOL_GPL(xenbus_unmap_ring_vfree);
687
688#ifdef CONFIG_XEN_PV
689static int xenbus_map_ring_pv(struct xenbus_device *dev,
690 struct map_ring_valloc *info,
691 grant_ref_t *gnt_refs,
692 unsigned int nr_grefs,
693 void **vaddr)
694{
695 struct xenbus_map_node *node = info->node;
696 struct vm_struct *area;
697 int err = GNTST_okay;
698 int i;
699 bool leaked;
700
701 area = alloc_vm_area(XEN_PAGE_SIZE * nr_grefs, info->ptes);
702 if (!area)
703 return -ENOMEM;
704
705 for (i = 0; i < nr_grefs; i++)
706 info->phys_addrs[i] =
707 arbitrary_virt_to_machine(info->ptes[i]).maddr;
708
709 err = __xenbus_map_ring(dev, gnt_refs, nr_grefs, node->handles,
710 info, GNTMAP_host_map | GNTMAP_contains_pte,
711 &leaked);
712 if (err)
713 goto failed;
714
715 node->nr_handles = nr_grefs;
716 node->pv.area = area;
717
718 spin_lock(&xenbus_valloc_lock);
719 list_add(&node->next, &xenbus_valloc_pages);
720 spin_unlock(&xenbus_valloc_lock);
721
722 *vaddr = area->addr;
723 info->node = NULL;
724
725 return 0;
726
727failed:
728 if (!leaked)
729 free_vm_area(area);
730 else
731 pr_alert("leaking VM area %p size %u page(s)", area, nr_grefs);
732
733 return err;
734}
735
736static int xenbus_unmap_ring_pv(struct xenbus_device *dev, void *vaddr)
737{
738 struct xenbus_map_node *node;
739 struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_GRANTS];
740 unsigned int level;
741 int i;
742 bool leaked = false;
743 int err;
744
745 spin_lock(&xenbus_valloc_lock);
746 list_for_each_entry(node, &xenbus_valloc_pages, next) {
747 if (node->pv.area->addr == vaddr) {
748 list_del(&node->next);
749 goto found;
750 }
751 }
752 node = NULL;
753 found:
754 spin_unlock(&xenbus_valloc_lock);
755
756 if (!node) {
757 xenbus_dev_error(dev, -ENOENT,
758 "can't find mapped virtual address %p", vaddr);
759 return GNTST_bad_virt_addr;
760 }
761
762 for (i = 0; i < node->nr_handles; i++) {
763 unsigned long addr;
764
765 memset(&unmap[i], 0, sizeof(unmap[i]));
766 addr = (unsigned long)vaddr + (XEN_PAGE_SIZE * i);
767 unmap[i].host_addr = arbitrary_virt_to_machine(
768 lookup_address(addr, &level)).maddr;
769 unmap[i].dev_bus_addr = 0;
770 unmap[i].handle = node->handles[i];
771 }
772
773 if (HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap, i))
774 BUG();
775
776 err = GNTST_okay;
777 leaked = false;
778 for (i = 0; i < node->nr_handles; i++) {
779 if (unmap[i].status != GNTST_okay) {
780 leaked = true;
781 xenbus_dev_error(dev, unmap[i].status,
782 "unmapping page at handle %d error %d",
783 node->handles[i], unmap[i].status);
784 err = unmap[i].status;
785 break;
786 }
787 }
788
789 if (!leaked)
790 free_vm_area(node->pv.area);
791 else
792 pr_alert("leaking VM area %p size %u page(s)",
793 node->pv.area, node->nr_handles);
794
795 kfree(node);
796 return err;
797}
798
799static const struct xenbus_ring_ops ring_ops_pv = {
800 .map = xenbus_map_ring_pv,
801 .unmap = xenbus_unmap_ring_pv,
802};
803#endif
804
805struct unmap_ring_hvm
806{
807 unsigned int idx;
808 unsigned long addrs[XENBUS_MAX_RING_GRANTS];
809};
810
811static void xenbus_unmap_ring_setup_grant_hvm(unsigned long gfn,
812 unsigned int goffset,
813 unsigned int len,
814 void *data)
815{
816 struct unmap_ring_hvm *info = data;
817
818 info->addrs[info->idx] = (unsigned long)gfn_to_virt(gfn);
819
820 info->idx++;
821}
822
823static int xenbus_unmap_ring_hvm(struct xenbus_device *dev, void *vaddr)
824{
825 int rv;
826 struct xenbus_map_node *node;
827 void *addr;
828 struct unmap_ring_hvm info = {
829 .idx = 0,
830 };
831 unsigned int nr_pages;
832
833 spin_lock(&xenbus_valloc_lock);
834 list_for_each_entry(node, &xenbus_valloc_pages, next) {
835 addr = node->hvm.addr;
836 if (addr == vaddr) {
837 list_del(&node->next);
838 goto found;
839 }
840 }
841 node = addr = NULL;
842 found:
843 spin_unlock(&xenbus_valloc_lock);
844
845 if (!node) {
846 xenbus_dev_error(dev, -ENOENT,
847 "can't find mapped virtual address %p", vaddr);
848 return GNTST_bad_virt_addr;
849 }
850
851 nr_pages = XENBUS_PAGES(node->nr_handles);
852
853 gnttab_foreach_grant(node->hvm.pages, node->nr_handles,
854 xenbus_unmap_ring_setup_grant_hvm,
855 &info);
856
857 rv = xenbus_unmap_ring(dev, node->handles, node->nr_handles,
858 info.addrs);
859 if (!rv) {
860 vunmap(vaddr);
861 xen_free_unpopulated_pages(nr_pages, node->hvm.pages);
862 }
863 else
864 WARN(1, "Leaking %p, size %u page(s)\n", vaddr, nr_pages);
865
866 kfree(node);
867 return rv;
868}
869
870
871
872
873
874
875
876
877enum xenbus_state xenbus_read_driver_state(const char *path)
878{
879 enum xenbus_state result;
880 int err = xenbus_gather(XBT_NIL, path, "state", "%d", &result, NULL);
881 if (err)
882 result = XenbusStateUnknown;
883
884 return result;
885}
886EXPORT_SYMBOL_GPL(xenbus_read_driver_state);
887
888static const struct xenbus_ring_ops ring_ops_hvm = {
889 .map = xenbus_map_ring_hvm,
890 .unmap = xenbus_unmap_ring_hvm,
891};
892
893void __init xenbus_ring_ops_init(void)
894{
895#ifdef CONFIG_XEN_PV
896 if (!xen_feature(XENFEAT_auto_translated_physmap))
897 ring_ops = &ring_ops_pv;
898 else
899#endif
900 ring_ops = &ring_ops_hvm;
901}
902