1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59#define pr_fmt(fmt) "virtio-mmio: " fmt
60
61#include <linux/highmem.h>
62#include <linux/interrupt.h>
63#include <linux/io.h>
64#include <linux/list.h>
65#include <linux/module.h>
66#include <linux/platform_device.h>
67#include <linux/slab.h>
68#include <linux/spinlock.h>
69#include <linux/virtio.h>
70#include <linux/virtio_config.h>
71#include <linux/virtio_mmio.h>
72#include <linux/virtio_ring.h>
73
74
75
76
77
78#define VIRTIO_MMIO_VRING_ALIGN PAGE_SIZE
79
80
81
82#define to_virtio_mmio_device(_plat_dev) \
83 container_of(_plat_dev, struct virtio_mmio_device, vdev)
84
85struct virtio_mmio_device {
86 struct virtio_device vdev;
87 struct platform_device *pdev;
88
89 void __iomem *base;
90 unsigned long version;
91
92
93 spinlock_t lock;
94 struct list_head virtqueues;
95};
96
97struct virtio_mmio_vq_info {
98
99 struct virtqueue *vq;
100
101
102 unsigned int num;
103
104
105 void *queue;
106
107
108 struct list_head node;
109};
110
111
112
113
114
115static u64 vm_get_features(struct virtio_device *vdev)
116{
117 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
118 u64 features;
119
120 writel(1, vm_dev->base + VIRTIO_MMIO_DEVICE_FEATURES_SEL);
121 features = readl(vm_dev->base + VIRTIO_MMIO_DEVICE_FEATURES);
122 features <<= 32;
123
124 writel(0, vm_dev->base + VIRTIO_MMIO_DEVICE_FEATURES_SEL);
125 features |= readl(vm_dev->base + VIRTIO_MMIO_DEVICE_FEATURES);
126
127 return features;
128}
129
130static int vm_finalize_features(struct virtio_device *vdev)
131{
132 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
133
134
135 vring_transport_features(vdev);
136
137
138 if (vm_dev->version == 2 &&
139 !__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) {
140 dev_err(&vdev->dev, "New virtio-mmio devices (version 2) must provide VIRTIO_F_VERSION_1 feature!\n");
141 return -EINVAL;
142 }
143
144 writel(1, vm_dev->base + VIRTIO_MMIO_DRIVER_FEATURES_SEL);
145 writel((u32)(vdev->features >> 32),
146 vm_dev->base + VIRTIO_MMIO_DRIVER_FEATURES);
147
148 writel(0, vm_dev->base + VIRTIO_MMIO_DRIVER_FEATURES_SEL);
149 writel((u32)vdev->features,
150 vm_dev->base + VIRTIO_MMIO_DRIVER_FEATURES);
151
152 return 0;
153}
154
155static void vm_get(struct virtio_device *vdev, unsigned offset,
156 void *buf, unsigned len)
157{
158 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
159 void __iomem *base = vm_dev->base + VIRTIO_MMIO_CONFIG;
160 u8 b;
161 __le16 w;
162 __le32 l;
163
164 if (vm_dev->version == 1) {
165 u8 *ptr = buf;
166 int i;
167
168 for (i = 0; i < len; i++)
169 ptr[i] = readb(base + offset + i);
170 return;
171 }
172
173 switch (len) {
174 case 1:
175 b = readb(base + offset);
176 memcpy(buf, &b, sizeof b);
177 break;
178 case 2:
179 w = cpu_to_le16(readw(base + offset));
180 memcpy(buf, &w, sizeof w);
181 break;
182 case 4:
183 l = cpu_to_le32(readl(base + offset));
184 memcpy(buf, &l, sizeof l);
185 break;
186 case 8:
187 l = cpu_to_le32(readl(base + offset));
188 memcpy(buf, &l, sizeof l);
189 l = cpu_to_le32(ioread32(base + offset + sizeof l));
190 memcpy(buf + sizeof l, &l, sizeof l);
191 break;
192 default:
193 BUG();
194 }
195}
196
197static void vm_set(struct virtio_device *vdev, unsigned offset,
198 const void *buf, unsigned len)
199{
200 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
201 void __iomem *base = vm_dev->base + VIRTIO_MMIO_CONFIG;
202 u8 b;
203 __le16 w;
204 __le32 l;
205
206 if (vm_dev->version == 1) {
207 const u8 *ptr = buf;
208 int i;
209
210 for (i = 0; i < len; i++)
211 writeb(ptr[i], base + offset + i);
212
213 return;
214 }
215
216 switch (len) {
217 case 1:
218 memcpy(&b, buf, sizeof b);
219 writeb(b, base + offset);
220 break;
221 case 2:
222 memcpy(&w, buf, sizeof w);
223 writew(le16_to_cpu(w), base + offset);
224 break;
225 case 4:
226 memcpy(&l, buf, sizeof l);
227 writel(le32_to_cpu(l), base + offset);
228 break;
229 case 8:
230 memcpy(&l, buf, sizeof l);
231 writel(le32_to_cpu(l), base + offset);
232 memcpy(&l, buf + sizeof l, sizeof l);
233 writel(le32_to_cpu(l), base + offset + sizeof l);
234 break;
235 default:
236 BUG();
237 }
238}
239
240static u32 vm_generation(struct virtio_device *vdev)
241{
242 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
243
244 if (vm_dev->version == 1)
245 return 0;
246 else
247 return readl(vm_dev->base + VIRTIO_MMIO_CONFIG_GENERATION);
248}
249
250static u8 vm_get_status(struct virtio_device *vdev)
251{
252 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
253
254 return readl(vm_dev->base + VIRTIO_MMIO_STATUS) & 0xff;
255}
256
257static void vm_set_status(struct virtio_device *vdev, u8 status)
258{
259 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
260
261
262 BUG_ON(status == 0);
263
264 writel(status, vm_dev->base + VIRTIO_MMIO_STATUS);
265}
266
267static void vm_reset(struct virtio_device *vdev)
268{
269 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
270
271
272 writel(0, vm_dev->base + VIRTIO_MMIO_STATUS);
273}
274
275
276
277
278
279
280static bool vm_notify(struct virtqueue *vq)
281{
282 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev);
283
284
285
286 writel(vq->index, vm_dev->base + VIRTIO_MMIO_QUEUE_NOTIFY);
287 return true;
288}
289
290
291static irqreturn_t vm_interrupt(int irq, void *opaque)
292{
293 struct virtio_mmio_device *vm_dev = opaque;
294 struct virtio_mmio_vq_info *info;
295 unsigned long status;
296 unsigned long flags;
297 irqreturn_t ret = IRQ_NONE;
298
299
300 status = readl(vm_dev->base + VIRTIO_MMIO_INTERRUPT_STATUS);
301 writel(status, vm_dev->base + VIRTIO_MMIO_INTERRUPT_ACK);
302
303 if (unlikely(status & VIRTIO_MMIO_INT_CONFIG)) {
304 virtio_config_changed(&vm_dev->vdev);
305 ret = IRQ_HANDLED;
306 }
307
308 if (likely(status & VIRTIO_MMIO_INT_VRING)) {
309 spin_lock_irqsave(&vm_dev->lock, flags);
310 list_for_each_entry(info, &vm_dev->virtqueues, node)
311 ret |= vring_interrupt(irq, info->vq);
312 spin_unlock_irqrestore(&vm_dev->lock, flags);
313 }
314
315 return ret;
316}
317
318
319
320static void vm_del_vq(struct virtqueue *vq)
321{
322 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev);
323 struct virtio_mmio_vq_info *info = vq->priv;
324 unsigned long flags, size;
325 unsigned int index = vq->index;
326
327 spin_lock_irqsave(&vm_dev->lock, flags);
328 list_del(&info->node);
329 spin_unlock_irqrestore(&vm_dev->lock, flags);
330
331 vring_del_virtqueue(vq);
332
333
334 writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL);
335 if (vm_dev->version == 1) {
336 writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
337 } else {
338 writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_READY);
339 WARN_ON(readl(vm_dev->base + VIRTIO_MMIO_QUEUE_READY));
340 }
341
342 size = PAGE_ALIGN(vring_size(info->num, VIRTIO_MMIO_VRING_ALIGN));
343 free_pages_exact(info->queue, size);
344 kfree(info);
345}
346
347static void vm_del_vqs(struct virtio_device *vdev)
348{
349 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
350 struct virtqueue *vq, *n;
351
352 list_for_each_entry_safe(vq, n, &vdev->vqs, list)
353 vm_del_vq(vq);
354
355 free_irq(platform_get_irq(vm_dev->pdev, 0), vm_dev);
356}
357
358
359
360static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned index,
361 void (*callback)(struct virtqueue *vq),
362 const char *name)
363{
364 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
365 struct virtio_mmio_vq_info *info;
366 struct virtqueue *vq;
367 unsigned long flags, size;
368 int err;
369
370 if (!name)
371 return NULL;
372
373
374 writel(index, vm_dev->base + VIRTIO_MMIO_QUEUE_SEL);
375
376
377 if (readl(vm_dev->base + (vm_dev->version == 1 ?
378 VIRTIO_MMIO_QUEUE_PFN : VIRTIO_MMIO_QUEUE_READY))) {
379 err = -ENOENT;
380 goto error_available;
381 }
382
383
384 info = kmalloc(sizeof(*info), GFP_KERNEL);
385 if (!info) {
386 err = -ENOMEM;
387 goto error_kmalloc;
388 }
389
390
391
392
393
394
395 info->num = readl(vm_dev->base + VIRTIO_MMIO_QUEUE_NUM_MAX);
396
397
398
399
400
401 if (info->num == 0) {
402 err = -ENOENT;
403 goto error_alloc_pages;
404 }
405
406 while (1) {
407 size = PAGE_ALIGN(vring_size(info->num,
408 VIRTIO_MMIO_VRING_ALIGN));
409
410 if (size < VIRTIO_MMIO_VRING_ALIGN * 2) {
411 err = -ENOMEM;
412 goto error_alloc_pages;
413 }
414
415 info->queue = alloc_pages_exact(size, GFP_KERNEL | __GFP_ZERO);
416 if (info->queue)
417 break;
418
419 info->num /= 2;
420 }
421
422
423 vq = vring_new_virtqueue(index, info->num, VIRTIO_MMIO_VRING_ALIGN, vdev,
424 true, info->queue, vm_notify, callback, name);
425 if (!vq) {
426 err = -ENOMEM;
427 goto error_new_virtqueue;
428 }
429
430
431 writel(info->num, vm_dev->base + VIRTIO_MMIO_QUEUE_NUM);
432 if (vm_dev->version == 1) {
433 writel(PAGE_SIZE, vm_dev->base + VIRTIO_MMIO_QUEUE_ALIGN);
434 writel(virt_to_phys(info->queue) >> PAGE_SHIFT,
435 vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
436 } else {
437 u64 addr;
438
439 addr = virt_to_phys(info->queue);
440 writel((u32)addr, vm_dev->base + VIRTIO_MMIO_QUEUE_DESC_LOW);
441 writel((u32)(addr >> 32),
442 vm_dev->base + VIRTIO_MMIO_QUEUE_DESC_HIGH);
443
444 addr = virt_to_phys(virtqueue_get_avail(vq));
445 writel((u32)addr, vm_dev->base + VIRTIO_MMIO_QUEUE_AVAIL_LOW);
446 writel((u32)(addr >> 32),
447 vm_dev->base + VIRTIO_MMIO_QUEUE_AVAIL_HIGH);
448
449 addr = virt_to_phys(virtqueue_get_used(vq));
450 writel((u32)addr, vm_dev->base + VIRTIO_MMIO_QUEUE_USED_LOW);
451 writel((u32)(addr >> 32),
452 vm_dev->base + VIRTIO_MMIO_QUEUE_USED_HIGH);
453
454 writel(1, vm_dev->base + VIRTIO_MMIO_QUEUE_READY);
455 }
456
457 vq->priv = info;
458 info->vq = vq;
459
460 spin_lock_irqsave(&vm_dev->lock, flags);
461 list_add(&info->node, &vm_dev->virtqueues);
462 spin_unlock_irqrestore(&vm_dev->lock, flags);
463
464 return vq;
465
466error_new_virtqueue:
467 if (vm_dev->version == 1) {
468 writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_PFN);
469 } else {
470 writel(0, vm_dev->base + VIRTIO_MMIO_QUEUE_READY);
471 WARN_ON(readl(vm_dev->base + VIRTIO_MMIO_QUEUE_READY));
472 }
473 free_pages_exact(info->queue, size);
474error_alloc_pages:
475 kfree(info);
476error_kmalloc:
477error_available:
478 return ERR_PTR(err);
479}
480
481static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs,
482 struct virtqueue *vqs[],
483 vq_callback_t *callbacks[],
484 const char *names[])
485{
486 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
487 unsigned int irq = platform_get_irq(vm_dev->pdev, 0);
488 int i, err;
489
490 err = request_irq(irq, vm_interrupt, IRQF_SHARED,
491 dev_name(&vdev->dev), vm_dev);
492 if (err)
493 return err;
494
495 for (i = 0; i < nvqs; ++i) {
496 vqs[i] = vm_setup_vq(vdev, i, callbacks[i], names[i]);
497 if (IS_ERR(vqs[i])) {
498 vm_del_vqs(vdev);
499 return PTR_ERR(vqs[i]);
500 }
501 }
502
503 return 0;
504}
505
506static const char *vm_bus_name(struct virtio_device *vdev)
507{
508 struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev);
509
510 return vm_dev->pdev->name;
511}
512
513static const struct virtio_config_ops virtio_mmio_config_ops = {
514 .get = vm_get,
515 .set = vm_set,
516 .generation = vm_generation,
517 .get_status = vm_get_status,
518 .set_status = vm_set_status,
519 .reset = vm_reset,
520 .find_vqs = vm_find_vqs,
521 .del_vqs = vm_del_vqs,
522 .get_features = vm_get_features,
523 .finalize_features = vm_finalize_features,
524 .bus_name = vm_bus_name,
525};
526
527
528
529
530
531static int virtio_mmio_probe(struct platform_device *pdev)
532{
533 struct virtio_mmio_device *vm_dev;
534 struct resource *mem;
535 unsigned long magic;
536
537 mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
538 if (!mem)
539 return -EINVAL;
540
541 if (!devm_request_mem_region(&pdev->dev, mem->start,
542 resource_size(mem), pdev->name))
543 return -EBUSY;
544
545 vm_dev = devm_kzalloc(&pdev->dev, sizeof(*vm_dev), GFP_KERNEL);
546 if (!vm_dev)
547 return -ENOMEM;
548
549 vm_dev->vdev.dev.parent = &pdev->dev;
550 vm_dev->vdev.config = &virtio_mmio_config_ops;
551 vm_dev->pdev = pdev;
552 INIT_LIST_HEAD(&vm_dev->virtqueues);
553 spin_lock_init(&vm_dev->lock);
554
555 vm_dev->base = devm_ioremap(&pdev->dev, mem->start, resource_size(mem));
556 if (vm_dev->base == NULL)
557 return -EFAULT;
558
559
560 magic = readl(vm_dev->base + VIRTIO_MMIO_MAGIC_VALUE);
561 if (magic != ('v' | 'i' << 8 | 'r' << 16 | 't' << 24)) {
562 dev_warn(&pdev->dev, "Wrong magic value 0x%08lx!\n", magic);
563 return -ENODEV;
564 }
565
566
567 vm_dev->version = readl(vm_dev->base + VIRTIO_MMIO_VERSION);
568 if (vm_dev->version < 1 || vm_dev->version > 2) {
569 dev_err(&pdev->dev, "Version %ld not supported!\n",
570 vm_dev->version);
571 return -ENXIO;
572 }
573
574 vm_dev->vdev.id.device = readl(vm_dev->base + VIRTIO_MMIO_DEVICE_ID);
575 if (vm_dev->vdev.id.device == 0) {
576
577
578
579
580 return -ENODEV;
581 }
582 vm_dev->vdev.id.vendor = readl(vm_dev->base + VIRTIO_MMIO_VENDOR_ID);
583
584
585 if (vm_dev->version == 2 &&
586 virtio_device_is_legacy_only(vm_dev->vdev.id)) {
587 dev_err(&pdev->dev, "Version 2 not supported for devices %u!\n",
588 vm_dev->vdev.id.device);
589 return -ENODEV;
590 }
591
592 if (vm_dev->version == 1)
593 writel(PAGE_SIZE, vm_dev->base + VIRTIO_MMIO_GUEST_PAGE_SIZE);
594
595 platform_set_drvdata(pdev, vm_dev);
596
597 return register_virtio_device(&vm_dev->vdev);
598}
599
600static int virtio_mmio_remove(struct platform_device *pdev)
601{
602 struct virtio_mmio_device *vm_dev = platform_get_drvdata(pdev);
603
604 unregister_virtio_device(&vm_dev->vdev);
605
606 return 0;
607}
608
609
610
611
612
613#if defined(CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES)
614
615static struct device vm_cmdline_parent = {
616 .init_name = "virtio-mmio-cmdline",
617};
618
619static int vm_cmdline_parent_registered;
620static int vm_cmdline_id;
621
622static int vm_cmdline_set(const char *device,
623 const struct kernel_param *kp)
624{
625 int err;
626 struct resource resources[2] = {};
627 char *str;
628 long long int base, size;
629 unsigned int irq;
630 int processed, consumed = 0;
631 struct platform_device *pdev;
632
633
634 size = memparse(device, &str);
635
636
637 processed = sscanf(str, "@%lli:%u%n:%d%n",
638 &base, &irq, &consumed,
639 &vm_cmdline_id, &consumed);
640
641
642
643
644
645
646 if (processed < 2 || str[consumed])
647 return -EINVAL;
648
649 resources[0].flags = IORESOURCE_MEM;
650 resources[0].start = base;
651 resources[0].end = base + size - 1;
652
653 resources[1].flags = IORESOURCE_IRQ;
654 resources[1].start = resources[1].end = irq;
655
656 if (!vm_cmdline_parent_registered) {
657 err = device_register(&vm_cmdline_parent);
658 if (err) {
659 pr_err("Failed to register parent device!\n");
660 return err;
661 }
662 vm_cmdline_parent_registered = 1;
663 }
664
665 pr_info("Registering device virtio-mmio.%d at 0x%llx-0x%llx, IRQ %d.\n",
666 vm_cmdline_id,
667 (unsigned long long)resources[0].start,
668 (unsigned long long)resources[0].end,
669 (int)resources[1].start);
670
671 pdev = platform_device_register_resndata(&vm_cmdline_parent,
672 "virtio-mmio", vm_cmdline_id++,
673 resources, ARRAY_SIZE(resources), NULL, 0);
674 if (IS_ERR(pdev))
675 return PTR_ERR(pdev);
676
677 return 0;
678}
679
680static int vm_cmdline_get_device(struct device *dev, void *data)
681{
682 char *buffer = data;
683 unsigned int len = strlen(buffer);
684 struct platform_device *pdev = to_platform_device(dev);
685
686 snprintf(buffer + len, PAGE_SIZE - len, "0x%llx@0x%llx:%llu:%d\n",
687 pdev->resource[0].end - pdev->resource[0].start + 1ULL,
688 (unsigned long long)pdev->resource[0].start,
689 (unsigned long long)pdev->resource[1].start,
690 pdev->id);
691 return 0;
692}
693
694static int vm_cmdline_get(char *buffer, const struct kernel_param *kp)
695{
696 buffer[0] = '\0';
697 device_for_each_child(&vm_cmdline_parent, buffer,
698 vm_cmdline_get_device);
699 return strlen(buffer) + 1;
700}
701
702static struct kernel_param_ops vm_cmdline_param_ops = {
703 .set = vm_cmdline_set,
704 .get = vm_cmdline_get,
705};
706
707device_param_cb(device, &vm_cmdline_param_ops, NULL, S_IRUSR);
708
709static int vm_unregister_cmdline_device(struct device *dev,
710 void *data)
711{
712 platform_device_unregister(to_platform_device(dev));
713
714 return 0;
715}
716
717static void vm_unregister_cmdline_devices(void)
718{
719 if (vm_cmdline_parent_registered) {
720 device_for_each_child(&vm_cmdline_parent, NULL,
721 vm_unregister_cmdline_device);
722 device_unregister(&vm_cmdline_parent);
723 vm_cmdline_parent_registered = 0;
724 }
725}
726
727#else
728
729static void vm_unregister_cmdline_devices(void)
730{
731}
732
733#endif
734
735
736
737static struct of_device_id virtio_mmio_match[] = {
738 { .compatible = "virtio,mmio", },
739 {},
740};
741MODULE_DEVICE_TABLE(of, virtio_mmio_match);
742
743static struct platform_driver virtio_mmio_driver = {
744 .probe = virtio_mmio_probe,
745 .remove = virtio_mmio_remove,
746 .driver = {
747 .name = "virtio-mmio",
748 .of_match_table = virtio_mmio_match,
749 },
750};
751
752static int __init virtio_mmio_init(void)
753{
754 return platform_driver_register(&virtio_mmio_driver);
755}
756
757static void __exit virtio_mmio_exit(void)
758{
759 platform_driver_unregister(&virtio_mmio_driver);
760 vm_unregister_cmdline_devices();
761}
762
763module_init(virtio_mmio_init);
764module_exit(virtio_mmio_exit);
765
766MODULE_AUTHOR("Pawel Moll <pawel.moll@arm.com>");
767MODULE_DESCRIPTION("Platform bus driver for memory mapped virtio devices");
768MODULE_LICENSE("GPL");
769