1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/mm.h>
34#include <linux/slab.h>
35#include <linux/types.h>
36#include <linux/spinlock.h>
37#include <linux/vmalloc.h>
38#include <linux/export.h>
39#include <asm/xen/hypervisor.h>
40#include <xen/page.h>
41#include <xen/interface/xen.h>
42#include <xen/interface/event_channel.h>
43#include <xen/balloon.h>
44#include <xen/events.h>
45#include <xen/grant_table.h>
46#include <xen/xenbus.h>
47#include <xen/xen.h>
48#include <xen/features.h>
49
50#include "xenbus.h"
51
52#define XENBUS_PAGES(_grants) (DIV_ROUND_UP(_grants, XEN_PFN_PER_PAGE))
53
54#define XENBUS_MAX_RING_PAGES (XENBUS_PAGES(XENBUS_MAX_RING_GRANTS))
55
56struct xenbus_map_node {
57 struct list_head next;
58 union {
59 struct {
60 struct vm_struct *area;
61 } pv;
62 struct {
63 struct page *pages[XENBUS_MAX_RING_PAGES];
64 unsigned long addrs[XENBUS_MAX_RING_GRANTS];
65 void *addr;
66 } hvm;
67 };
68 grant_handle_t handles[XENBUS_MAX_RING_GRANTS];
69 unsigned int nr_handles;
70};
71
72struct map_ring_valloc {
73 struct xenbus_map_node *node;
74
75
76 unsigned long addrs[XENBUS_MAX_RING_GRANTS];
77 phys_addr_t phys_addrs[XENBUS_MAX_RING_GRANTS];
78
79 struct gnttab_map_grant_ref map[XENBUS_MAX_RING_GRANTS];
80 struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_GRANTS];
81
82 unsigned int idx;
83};
84
85static DEFINE_SPINLOCK(xenbus_valloc_lock);
86static LIST_HEAD(xenbus_valloc_pages);
87
88struct xenbus_ring_ops {
89 int (*map)(struct xenbus_device *dev, struct map_ring_valloc *info,
90 grant_ref_t *gnt_refs, unsigned int nr_grefs,
91 void **vaddr);
92 int (*unmap)(struct xenbus_device *dev, void *vaddr);
93};
94
95static const struct xenbus_ring_ops *ring_ops __read_mostly;
96
97const char *xenbus_strstate(enum xenbus_state state)
98{
99 static const char *const name[] = {
100 [ XenbusStateUnknown ] = "Unknown",
101 [ XenbusStateInitialising ] = "Initialising",
102 [ XenbusStateInitWait ] = "InitWait",
103 [ XenbusStateInitialised ] = "Initialised",
104 [ XenbusStateConnected ] = "Connected",
105 [ XenbusStateClosing ] = "Closing",
106 [ XenbusStateClosed ] = "Closed",
107 [XenbusStateReconfiguring] = "Reconfiguring",
108 [XenbusStateReconfigured] = "Reconfigured",
109 };
110 return (state < ARRAY_SIZE(name)) ? name[state] : "INVALID";
111}
112EXPORT_SYMBOL_GPL(xenbus_strstate);
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128int xenbus_watch_path(struct xenbus_device *dev, const char *path,
129 struct xenbus_watch *watch,
130 bool (*will_handle)(struct xenbus_watch *,
131 const char *, const char *),
132 void (*callback)(struct xenbus_watch *,
133 const char *, const char *))
134{
135 int err;
136
137 watch->node = path;
138 watch->will_handle = will_handle;
139 watch->callback = callback;
140
141 err = register_xenbus_watch(watch);
142
143 if (err) {
144 watch->node = NULL;
145 watch->will_handle = NULL;
146 watch->callback = NULL;
147 xenbus_dev_fatal(dev, err, "adding watch on %s", path);
148 }
149
150 return err;
151}
152EXPORT_SYMBOL_GPL(xenbus_watch_path);
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170int xenbus_watch_pathfmt(struct xenbus_device *dev,
171 struct xenbus_watch *watch,
172 bool (*will_handle)(struct xenbus_watch *,
173 const char *, const char *),
174 void (*callback)(struct xenbus_watch *,
175 const char *, const char *),
176 const char *pathfmt, ...)
177{
178 int err;
179 va_list ap;
180 char *path;
181
182 va_start(ap, pathfmt);
183 path = kvasprintf(GFP_NOIO | __GFP_HIGH, pathfmt, ap);
184 va_end(ap);
185
186 if (!path) {
187 xenbus_dev_fatal(dev, -ENOMEM, "allocating path for watch");
188 return -ENOMEM;
189 }
190 err = xenbus_watch_path(dev, path, watch, will_handle, callback);
191
192 if (err)
193 kfree(path);
194 return err;
195}
196EXPORT_SYMBOL_GPL(xenbus_watch_pathfmt);
197
198static void xenbus_switch_fatal(struct xenbus_device *, int, int,
199 const char *, ...);
200
201static int
202__xenbus_switch_state(struct xenbus_device *dev,
203 enum xenbus_state state, int depth)
204{
205
206
207
208
209
210
211
212
213
214
215
216
217
218 struct xenbus_transaction xbt;
219 int current_state;
220 int err, abort;
221
222 if (state == dev->state)
223 return 0;
224
225again:
226 abort = 1;
227
228 err = xenbus_transaction_start(&xbt);
229 if (err) {
230 xenbus_switch_fatal(dev, depth, err, "starting transaction");
231 return 0;
232 }
233
234 err = xenbus_scanf(xbt, dev->nodename, "state", "%d", ¤t_state);
235 if (err != 1)
236 goto abort;
237
238 err = xenbus_printf(xbt, dev->nodename, "state", "%d", state);
239 if (err) {
240 xenbus_switch_fatal(dev, depth, err, "writing new state");
241 goto abort;
242 }
243
244 abort = 0;
245abort:
246 err = xenbus_transaction_end(xbt, abort);
247 if (err) {
248 if (err == -EAGAIN && !abort)
249 goto again;
250 xenbus_switch_fatal(dev, depth, err, "ending transaction");
251 } else
252 dev->state = state;
253
254 return 0;
255}
256
257
258
259
260
261
262
263
264
265
266int xenbus_switch_state(struct xenbus_device *dev, enum xenbus_state state)
267{
268 return __xenbus_switch_state(dev, state, 0);
269}
270
271EXPORT_SYMBOL_GPL(xenbus_switch_state);
272
273int xenbus_frontend_closed(struct xenbus_device *dev)
274{
275 xenbus_switch_state(dev, XenbusStateClosed);
276 complete(&dev->down);
277 return 0;
278}
279EXPORT_SYMBOL_GPL(xenbus_frontend_closed);
280
281static void xenbus_va_dev_error(struct xenbus_device *dev, int err,
282 const char *fmt, va_list ap)
283{
284 unsigned int len;
285 char *printf_buffer;
286 char *path_buffer;
287
288#define PRINTF_BUFFER_SIZE 4096
289
290 printf_buffer = kmalloc(PRINTF_BUFFER_SIZE, GFP_KERNEL);
291 if (!printf_buffer)
292 return;
293
294 len = sprintf(printf_buffer, "%i ", -err);
295 vsnprintf(printf_buffer + len, PRINTF_BUFFER_SIZE - len, fmt, ap);
296
297 dev_err(&dev->dev, "%s\n", printf_buffer);
298
299 path_buffer = kasprintf(GFP_KERNEL, "error/%s", dev->nodename);
300 if (path_buffer)
301 xenbus_write(XBT_NIL, path_buffer, "error", printf_buffer);
302
303 kfree(printf_buffer);
304 kfree(path_buffer);
305}
306
307
308
309
310
311
312
313
314
315
316void xenbus_dev_error(struct xenbus_device *dev, int err, const char *fmt, ...)
317{
318 va_list ap;
319
320 va_start(ap, fmt);
321 xenbus_va_dev_error(dev, err, fmt, ap);
322 va_end(ap);
323}
324EXPORT_SYMBOL_GPL(xenbus_dev_error);
325
326
327
328
329
330
331
332
333
334
335
336
337void xenbus_dev_fatal(struct xenbus_device *dev, int err, const char *fmt, ...)
338{
339 va_list ap;
340
341 va_start(ap, fmt);
342 xenbus_va_dev_error(dev, err, fmt, ap);
343 va_end(ap);
344
345 xenbus_switch_state(dev, XenbusStateClosing);
346}
347EXPORT_SYMBOL_GPL(xenbus_dev_fatal);
348
349
350
351
352
353static void xenbus_switch_fatal(struct xenbus_device *dev, int depth, int err,
354 const char *fmt, ...)
355{
356 va_list ap;
357
358 va_start(ap, fmt);
359 xenbus_va_dev_error(dev, err, fmt, ap);
360 va_end(ap);
361
362 if (!depth)
363 __xenbus_switch_state(dev, XenbusStateClosing, 1);
364}
365
366
367
368
369
370
371
372
373
374
375
376
377
378int xenbus_grant_ring(struct xenbus_device *dev, void *vaddr,
379 unsigned int nr_pages, grant_ref_t *grefs)
380{
381 int err;
382 int i, j;
383
384 for (i = 0; i < nr_pages; i++) {
385 unsigned long gfn;
386
387 if (is_vmalloc_addr(vaddr))
388 gfn = pfn_to_gfn(vmalloc_to_pfn(vaddr));
389 else
390 gfn = virt_to_gfn(vaddr);
391
392 err = gnttab_grant_foreign_access(dev->otherend_id, gfn, 0);
393 if (err < 0) {
394 xenbus_dev_fatal(dev, err,
395 "granting access to ring page");
396 goto fail;
397 }
398 grefs[i] = err;
399
400 vaddr = vaddr + XEN_PAGE_SIZE;
401 }
402
403 return 0;
404
405fail:
406 for (j = 0; j < i; j++)
407 gnttab_end_foreign_access_ref(grefs[j], 0);
408 return err;
409}
410EXPORT_SYMBOL_GPL(xenbus_grant_ring);
411
412
413
414
415
416
417
418
419int xenbus_alloc_evtchn(struct xenbus_device *dev, evtchn_port_t *port)
420{
421 struct evtchn_alloc_unbound alloc_unbound;
422 int err;
423
424 alloc_unbound.dom = DOMID_SELF;
425 alloc_unbound.remote_dom = dev->otherend_id;
426
427 err = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
428 &alloc_unbound);
429 if (err)
430 xenbus_dev_fatal(dev, err, "allocating event channel");
431 else
432 *port = alloc_unbound.port;
433
434 return err;
435}
436EXPORT_SYMBOL_GPL(xenbus_alloc_evtchn);
437
438
439
440
441
442int xenbus_free_evtchn(struct xenbus_device *dev, evtchn_port_t port)
443{
444 struct evtchn_close close;
445 int err;
446
447 close.port = port;
448
449 err = HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
450 if (err)
451 xenbus_dev_error(dev, err, "freeing event channel %u", port);
452
453 return err;
454}
455EXPORT_SYMBOL_GPL(xenbus_free_evtchn);
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472int xenbus_map_ring_valloc(struct xenbus_device *dev, grant_ref_t *gnt_refs,
473 unsigned int nr_grefs, void **vaddr)
474{
475 int err;
476 struct map_ring_valloc *info;
477
478 *vaddr = NULL;
479
480 if (nr_grefs > XENBUS_MAX_RING_GRANTS)
481 return -EINVAL;
482
483 info = kzalloc(sizeof(*info), GFP_KERNEL);
484 if (!info)
485 return -ENOMEM;
486
487 info->node = kzalloc(sizeof(*info->node), GFP_KERNEL);
488 if (!info->node)
489 err = -ENOMEM;
490 else
491 err = ring_ops->map(dev, info, gnt_refs, nr_grefs, vaddr);
492
493 kfree(info->node);
494 kfree(info);
495 return err;
496}
497EXPORT_SYMBOL_GPL(xenbus_map_ring_valloc);
498
499
500
501
502static int __xenbus_map_ring(struct xenbus_device *dev,
503 grant_ref_t *gnt_refs,
504 unsigned int nr_grefs,
505 grant_handle_t *handles,
506 struct map_ring_valloc *info,
507 unsigned int flags,
508 bool *leaked)
509{
510 int i, j;
511
512 if (nr_grefs > XENBUS_MAX_RING_GRANTS)
513 return -EINVAL;
514
515 for (i = 0; i < nr_grefs; i++) {
516 gnttab_set_map_op(&info->map[i], info->phys_addrs[i], flags,
517 gnt_refs[i], dev->otherend_id);
518 handles[i] = INVALID_GRANT_HANDLE;
519 }
520
521 gnttab_batch_map(info->map, i);
522
523 for (i = 0; i < nr_grefs; i++) {
524 if (info->map[i].status != GNTST_okay) {
525 xenbus_dev_fatal(dev, info->map[i].status,
526 "mapping in shared page %d from domain %d",
527 gnt_refs[i], dev->otherend_id);
528 goto fail;
529 } else
530 handles[i] = info->map[i].handle;
531 }
532
533 return 0;
534
535 fail:
536 for (i = j = 0; i < nr_grefs; i++) {
537 if (handles[i] != INVALID_GRANT_HANDLE) {
538 gnttab_set_unmap_op(&info->unmap[j],
539 info->phys_addrs[i],
540 GNTMAP_host_map, handles[i]);
541 j++;
542 }
543 }
544
545 BUG_ON(HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, info->unmap, j));
546
547 *leaked = false;
548 for (i = 0; i < j; i++) {
549 if (info->unmap[i].status != GNTST_okay) {
550 *leaked = true;
551 break;
552 }
553 }
554
555 return -ENOENT;
556}
557
558
559
560
561
562
563
564
565
566
567
568
569static int xenbus_unmap_ring(struct xenbus_device *dev, grant_handle_t *handles,
570 unsigned int nr_handles, unsigned long *vaddrs)
571{
572 struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_GRANTS];
573 int i;
574 int err;
575
576 if (nr_handles > XENBUS_MAX_RING_GRANTS)
577 return -EINVAL;
578
579 for (i = 0; i < nr_handles; i++)
580 gnttab_set_unmap_op(&unmap[i], vaddrs[i],
581 GNTMAP_host_map, handles[i]);
582
583 BUG_ON(HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap, i));
584
585 err = GNTST_okay;
586 for (i = 0; i < nr_handles; i++) {
587 if (unmap[i].status != GNTST_okay) {
588 xenbus_dev_error(dev, unmap[i].status,
589 "unmapping page at handle %d error %d",
590 handles[i], unmap[i].status);
591 err = unmap[i].status;
592 break;
593 }
594 }
595
596 return err;
597}
598
599static void xenbus_map_ring_setup_grant_hvm(unsigned long gfn,
600 unsigned int goffset,
601 unsigned int len,
602 void *data)
603{
604 struct map_ring_valloc *info = data;
605 unsigned long vaddr = (unsigned long)gfn_to_virt(gfn);
606
607 info->phys_addrs[info->idx] = vaddr;
608 info->addrs[info->idx] = vaddr;
609
610 info->idx++;
611}
612
613static int xenbus_map_ring_hvm(struct xenbus_device *dev,
614 struct map_ring_valloc *info,
615 grant_ref_t *gnt_ref,
616 unsigned int nr_grefs,
617 void **vaddr)
618{
619 struct xenbus_map_node *node = info->node;
620 int err;
621 void *addr;
622 bool leaked = false;
623 unsigned int nr_pages = XENBUS_PAGES(nr_grefs);
624
625 err = xen_alloc_unpopulated_pages(nr_pages, node->hvm.pages);
626 if (err)
627 goto out_err;
628
629 gnttab_foreach_grant(node->hvm.pages, nr_grefs,
630 xenbus_map_ring_setup_grant_hvm,
631 info);
632
633 err = __xenbus_map_ring(dev, gnt_ref, nr_grefs, node->handles,
634 info, GNTMAP_host_map, &leaked);
635 node->nr_handles = nr_grefs;
636
637 if (err)
638 goto out_free_ballooned_pages;
639
640 addr = vmap(node->hvm.pages, nr_pages, VM_MAP | VM_IOREMAP,
641 PAGE_KERNEL);
642 if (!addr) {
643 err = -ENOMEM;
644 goto out_xenbus_unmap_ring;
645 }
646
647 node->hvm.addr = addr;
648
649 spin_lock(&xenbus_valloc_lock);
650 list_add(&node->next, &xenbus_valloc_pages);
651 spin_unlock(&xenbus_valloc_lock);
652
653 *vaddr = addr;
654 info->node = NULL;
655
656 return 0;
657
658 out_xenbus_unmap_ring:
659 if (!leaked)
660 xenbus_unmap_ring(dev, node->handles, nr_grefs, info->addrs);
661 else
662 pr_alert("leaking %p size %u page(s)",
663 addr, nr_pages);
664 out_free_ballooned_pages:
665 if (!leaked)
666 xen_free_unpopulated_pages(nr_pages, node->hvm.pages);
667 out_err:
668 return err;
669}
670
671
672
673
674
675
676
677
678
679
680
681
682
683int xenbus_unmap_ring_vfree(struct xenbus_device *dev, void *vaddr)
684{
685 return ring_ops->unmap(dev, vaddr);
686}
687EXPORT_SYMBOL_GPL(xenbus_unmap_ring_vfree);
688
689#ifdef CONFIG_XEN_PV
690static int map_ring_apply(pte_t *pte, unsigned long addr, void *data)
691{
692 struct map_ring_valloc *info = data;
693
694 info->phys_addrs[info->idx++] = arbitrary_virt_to_machine(pte).maddr;
695 return 0;
696}
697
698static int xenbus_map_ring_pv(struct xenbus_device *dev,
699 struct map_ring_valloc *info,
700 grant_ref_t *gnt_refs,
701 unsigned int nr_grefs,
702 void **vaddr)
703{
704 struct xenbus_map_node *node = info->node;
705 struct vm_struct *area;
706 bool leaked = false;
707 int err = -ENOMEM;
708
709 area = get_vm_area(XEN_PAGE_SIZE * nr_grefs, VM_IOREMAP);
710 if (!area)
711 return -ENOMEM;
712 if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
713 XEN_PAGE_SIZE * nr_grefs, map_ring_apply, info))
714 goto failed;
715 err = __xenbus_map_ring(dev, gnt_refs, nr_grefs, node->handles,
716 info, GNTMAP_host_map | GNTMAP_contains_pte,
717 &leaked);
718 if (err)
719 goto failed;
720
721 node->nr_handles = nr_grefs;
722 node->pv.area = area;
723
724 spin_lock(&xenbus_valloc_lock);
725 list_add(&node->next, &xenbus_valloc_pages);
726 spin_unlock(&xenbus_valloc_lock);
727
728 *vaddr = area->addr;
729 info->node = NULL;
730
731 return 0;
732
733failed:
734 if (!leaked)
735 free_vm_area(area);
736 else
737 pr_alert("leaking VM area %p size %u page(s)", area, nr_grefs);
738
739 return err;
740}
741
742static int xenbus_unmap_ring_pv(struct xenbus_device *dev, void *vaddr)
743{
744 struct xenbus_map_node *node;
745 struct gnttab_unmap_grant_ref unmap[XENBUS_MAX_RING_GRANTS];
746 unsigned int level;
747 int i;
748 bool leaked = false;
749 int err;
750
751 spin_lock(&xenbus_valloc_lock);
752 list_for_each_entry(node, &xenbus_valloc_pages, next) {
753 if (node->pv.area->addr == vaddr) {
754 list_del(&node->next);
755 goto found;
756 }
757 }
758 node = NULL;
759 found:
760 spin_unlock(&xenbus_valloc_lock);
761
762 if (!node) {
763 xenbus_dev_error(dev, -ENOENT,
764 "can't find mapped virtual address %p", vaddr);
765 return GNTST_bad_virt_addr;
766 }
767
768 for (i = 0; i < node->nr_handles; i++) {
769 unsigned long addr;
770
771 memset(&unmap[i], 0, sizeof(unmap[i]));
772 addr = (unsigned long)vaddr + (XEN_PAGE_SIZE * i);
773 unmap[i].host_addr = arbitrary_virt_to_machine(
774 lookup_address(addr, &level)).maddr;
775 unmap[i].dev_bus_addr = 0;
776 unmap[i].handle = node->handles[i];
777 }
778
779 BUG_ON(HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap, i));
780
781 err = GNTST_okay;
782 leaked = false;
783 for (i = 0; i < node->nr_handles; i++) {
784 if (unmap[i].status != GNTST_okay) {
785 leaked = true;
786 xenbus_dev_error(dev, unmap[i].status,
787 "unmapping page at handle %d error %d",
788 node->handles[i], unmap[i].status);
789 err = unmap[i].status;
790 break;
791 }
792 }
793
794 if (!leaked)
795 free_vm_area(node->pv.area);
796 else
797 pr_alert("leaking VM area %p size %u page(s)",
798 node->pv.area, node->nr_handles);
799
800 kfree(node);
801 return err;
802}
803
804static const struct xenbus_ring_ops ring_ops_pv = {
805 .map = xenbus_map_ring_pv,
806 .unmap = xenbus_unmap_ring_pv,
807};
808#endif
809
810struct unmap_ring_hvm
811{
812 unsigned int idx;
813 unsigned long addrs[XENBUS_MAX_RING_GRANTS];
814};
815
816static void xenbus_unmap_ring_setup_grant_hvm(unsigned long gfn,
817 unsigned int goffset,
818 unsigned int len,
819 void *data)
820{
821 struct unmap_ring_hvm *info = data;
822
823 info->addrs[info->idx] = (unsigned long)gfn_to_virt(gfn);
824
825 info->idx++;
826}
827
828static int xenbus_unmap_ring_hvm(struct xenbus_device *dev, void *vaddr)
829{
830 int rv;
831 struct xenbus_map_node *node;
832 void *addr;
833 struct unmap_ring_hvm info = {
834 .idx = 0,
835 };
836 unsigned int nr_pages;
837
838 spin_lock(&xenbus_valloc_lock);
839 list_for_each_entry(node, &xenbus_valloc_pages, next) {
840 addr = node->hvm.addr;
841 if (addr == vaddr) {
842 list_del(&node->next);
843 goto found;
844 }
845 }
846 node = addr = NULL;
847 found:
848 spin_unlock(&xenbus_valloc_lock);
849
850 if (!node) {
851 xenbus_dev_error(dev, -ENOENT,
852 "can't find mapped virtual address %p", vaddr);
853 return GNTST_bad_virt_addr;
854 }
855
856 nr_pages = XENBUS_PAGES(node->nr_handles);
857
858 gnttab_foreach_grant(node->hvm.pages, node->nr_handles,
859 xenbus_unmap_ring_setup_grant_hvm,
860 &info);
861
862 rv = xenbus_unmap_ring(dev, node->handles, node->nr_handles,
863 info.addrs);
864 if (!rv) {
865 vunmap(vaddr);
866 xen_free_unpopulated_pages(nr_pages, node->hvm.pages);
867 }
868 else
869 WARN(1, "Leaking %p, size %u page(s)\n", vaddr, nr_pages);
870
871 kfree(node);
872 return rv;
873}
874
875
876
877
878
879
880
881
882enum xenbus_state xenbus_read_driver_state(const char *path)
883{
884 enum xenbus_state result;
885 int err = xenbus_gather(XBT_NIL, path, "state", "%d", &result, NULL);
886 if (err)
887 result = XenbusStateUnknown;
888
889 return result;
890}
891EXPORT_SYMBOL_GPL(xenbus_read_driver_state);
892
893static const struct xenbus_ring_ops ring_ops_hvm = {
894 .map = xenbus_map_ring_hvm,
895 .unmap = xenbus_unmap_ring_hvm,
896};
897
898void __init xenbus_ring_ops_init(void)
899{
900#ifdef CONFIG_XEN_PV
901 if (!xen_feature(XENFEAT_auto_translated_physmap))
902 ring_ops = &ring_ops_pv;
903 else
904#endif
905 ring_ops = &ring_ops_hvm;
906}
907