1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/delay.h>
34#include <linux/module.h>
35#include <linux/sched.h>
36#include <linux/dma-mapping.h>
37
38#include "vop_main.h"
39
40#define VOP_MAX_VRINGS 4
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58struct _vop_vdev {
59 struct virtio_device vdev;
60 struct mic_device_desc __iomem *desc;
61 struct mic_device_ctrl __iomem *dc;
62 struct vop_device *vpdev;
63 void __iomem *vr[VOP_MAX_VRINGS];
64 dma_addr_t used[VOP_MAX_VRINGS];
65 int used_size[VOP_MAX_VRINGS];
66 struct completion reset_done;
67 struct mic_irq *virtio_cookie;
68 int c2h_vdev_db;
69 int h2c_vdev_db;
70 int dnode;
71};
72
73#define to_vopvdev(vd) container_of(vd, struct _vop_vdev, vdev)
74
75#define _vop_aligned_desc_size(d) __mic_align(_vop_desc_size(d), 8)
76
77
78static inline struct device *_vop_dev(struct _vop_vdev *vdev)
79{
80 return vdev->vdev.dev.parent;
81}
82
83static inline unsigned _vop_desc_size(struct mic_device_desc __iomem *desc)
84{
85 return sizeof(*desc)
86 + ioread8(&desc->num_vq) * sizeof(struct mic_vqconfig)
87 + ioread8(&desc->feature_len) * 2
88 + ioread8(&desc->config_len);
89}
90
91static inline struct mic_vqconfig __iomem *
92_vop_vq_config(struct mic_device_desc __iomem *desc)
93{
94 return (struct mic_vqconfig __iomem *)(desc + 1);
95}
96
97static inline u8 __iomem *
98_vop_vq_features(struct mic_device_desc __iomem *desc)
99{
100 return (u8 __iomem *)(_vop_vq_config(desc) + ioread8(&desc->num_vq));
101}
102
103static inline u8 __iomem *
104_vop_vq_configspace(struct mic_device_desc __iomem *desc)
105{
106 return _vop_vq_features(desc) + ioread8(&desc->feature_len) * 2;
107}
108
109static inline unsigned
110_vop_total_desc_size(struct mic_device_desc __iomem *desc)
111{
112 return _vop_aligned_desc_size(desc) + sizeof(struct mic_device_ctrl);
113}
114
115
116static u64 vop_get_features(struct virtio_device *vdev)
117{
118 unsigned int i, bits;
119 u32 features = 0;
120 struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
121 u8 __iomem *in_features = _vop_vq_features(desc);
122 int feature_len = ioread8(&desc->feature_len);
123
124 bits = min_t(unsigned, feature_len, sizeof(vdev->features)) * 8;
125 for (i = 0; i < bits; i++)
126 if (ioread8(&in_features[i / 8]) & (BIT(i % 8)))
127 features |= BIT(i);
128
129 return features;
130}
131
132static int vop_finalize_features(struct virtio_device *vdev)
133{
134 unsigned int i, bits;
135 struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
136 u8 feature_len = ioread8(&desc->feature_len);
137
138 u8 __iomem *out_features =
139 _vop_vq_features(desc) + feature_len;
140
141
142 vring_transport_features(vdev);
143
144 memset_io(out_features, 0, feature_len);
145 bits = min_t(unsigned, feature_len,
146 sizeof(vdev->features)) * 8;
147 for (i = 0; i < bits; i++) {
148 if (__virtio_test_bit(vdev, i))
149 iowrite8(ioread8(&out_features[i / 8]) | (1 << (i % 8)),
150 &out_features[i / 8]);
151 }
152 return 0;
153}
154
155
156
157
158static void vop_get(struct virtio_device *vdev, unsigned int offset,
159 void *buf, unsigned len)
160{
161 struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
162
163 if (offset + len > ioread8(&desc->config_len))
164 return;
165 memcpy_fromio(buf, _vop_vq_configspace(desc) + offset, len);
166}
167
168static void vop_set(struct virtio_device *vdev, unsigned int offset,
169 const void *buf, unsigned len)
170{
171 struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
172
173 if (offset + len > ioread8(&desc->config_len))
174 return;
175 memcpy_toio(_vop_vq_configspace(desc) + offset, buf, len);
176}
177
178
179
180
181
182
183static u8 vop_get_status(struct virtio_device *vdev)
184{
185 return ioread8(&to_vopvdev(vdev)->desc->status);
186}
187
188static void vop_set_status(struct virtio_device *dev, u8 status)
189{
190 struct _vop_vdev *vdev = to_vopvdev(dev);
191 struct vop_device *vpdev = vdev->vpdev;
192
193 if (!status)
194 return;
195 iowrite8(status, &vdev->desc->status);
196 vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
197}
198
199
200static void vop_reset_inform_host(struct virtio_device *dev)
201{
202 struct _vop_vdev *vdev = to_vopvdev(dev);
203 struct mic_device_ctrl __iomem *dc = vdev->dc;
204 struct vop_device *vpdev = vdev->vpdev;
205 int retry;
206
207 iowrite8(0, &dc->host_ack);
208 iowrite8(1, &dc->vdev_reset);
209 vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
210
211
212 for (retry = 100; retry--;) {
213 if (ioread8(&dc->host_ack))
214 break;
215 msleep(100);
216 };
217
218 dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry);
219
220
221 iowrite8(0, &vdev->desc->status);
222}
223
224static void vop_reset(struct virtio_device *dev)
225{
226 struct _vop_vdev *vdev = to_vopvdev(dev);
227
228 dev_dbg(_vop_dev(vdev), "%s: virtio id %d\n",
229 __func__, dev->id.device);
230
231 vop_reset_inform_host(dev);
232 complete_all(&vdev->reset_done);
233}
234
235
236
237
238static bool vop_notify(struct virtqueue *vq)
239{
240 struct _vop_vdev *vdev = vq->priv;
241 struct vop_device *vpdev = vdev->vpdev;
242
243 vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
244 return true;
245}
246
247static void vop_del_vq(struct virtqueue *vq, int n)
248{
249 struct _vop_vdev *vdev = to_vopvdev(vq->vdev);
250 struct vring *vr = (struct vring *)(vq + 1);
251 struct vop_device *vpdev = vdev->vpdev;
252
253 dma_unmap_single(&vpdev->dev, vdev->used[n],
254 vdev->used_size[n], DMA_BIDIRECTIONAL);
255 free_pages((unsigned long)vr->used, get_order(vdev->used_size[n]));
256 vring_del_virtqueue(vq);
257 vpdev->hw_ops->iounmap(vpdev, vdev->vr[n]);
258 vdev->vr[n] = NULL;
259}
260
261static void vop_del_vqs(struct virtio_device *dev)
262{
263 struct _vop_vdev *vdev = to_vopvdev(dev);
264 struct virtqueue *vq, *n;
265 int idx = 0;
266
267 dev_dbg(_vop_dev(vdev), "%s\n", __func__);
268
269 list_for_each_entry_safe(vq, n, &dev->vqs, list)
270 vop_del_vq(vq, idx++);
271}
272
273
274
275
276
277
278static struct virtqueue *vop_find_vq(struct virtio_device *dev,
279 unsigned index,
280 void (*callback)(struct virtqueue *vq),
281 const char *name, bool ctx)
282{
283 struct _vop_vdev *vdev = to_vopvdev(dev);
284 struct vop_device *vpdev = vdev->vpdev;
285 struct mic_vqconfig __iomem *vqconfig;
286 struct mic_vqconfig config;
287 struct virtqueue *vq;
288 void __iomem *va;
289 struct _mic_vring_info __iomem *info;
290 void *used;
291 int vr_size, _vr_size, err, magic;
292 struct vring *vr;
293 u8 type = ioread8(&vdev->desc->type);
294
295 if (index >= ioread8(&vdev->desc->num_vq))
296 return ERR_PTR(-ENOENT);
297
298 if (!name)
299 return ERR_PTR(-ENOENT);
300
301
302 vqconfig = _vop_vq_config(vdev->desc) + index;
303 memcpy_fromio(&config, vqconfig, sizeof(config));
304 _vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN);
305 vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info));
306 va = vpdev->hw_ops->ioremap(vpdev, le64_to_cpu(config.address),
307 vr_size);
308 if (!va)
309 return ERR_PTR(-ENOMEM);
310 vdev->vr[index] = va;
311 memset_io(va, 0x0, _vr_size);
312 vq = vring_new_virtqueue(
313 index,
314 le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN,
315 dev,
316 false,
317 ctx,
318 (void __force *)va, vop_notify, callback, name);
319 if (!vq) {
320 err = -ENOMEM;
321 goto unmap;
322 }
323 info = va + _vr_size;
324 magic = ioread32(&info->magic);
325
326 if (WARN(magic != MIC_MAGIC + type + index, "magic mismatch")) {
327 err = -EIO;
328 goto unmap;
329 }
330
331
332 vdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 +
333 sizeof(struct vring_used_elem) *
334 le16_to_cpu(config.num));
335 used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
336 get_order(vdev->used_size[index]));
337 if (!used) {
338 err = -ENOMEM;
339 dev_err(_vop_dev(vdev), "%s %d err %d\n",
340 __func__, __LINE__, err);
341 goto del_vq;
342 }
343 vdev->used[index] = dma_map_single(&vpdev->dev, used,
344 vdev->used_size[index],
345 DMA_BIDIRECTIONAL);
346 if (dma_mapping_error(&vpdev->dev, vdev->used[index])) {
347 err = -ENOMEM;
348 dev_err(_vop_dev(vdev), "%s %d err %d\n",
349 __func__, __LINE__, err);
350 goto free_used;
351 }
352 writeq(vdev->used[index], &vqconfig->used_address);
353
354
355
356
357
358
359
360 vr = (struct vring *)(vq + 1);
361 vr->used = used;
362
363 vq->priv = vdev;
364 return vq;
365free_used:
366 free_pages((unsigned long)used,
367 get_order(vdev->used_size[index]));
368del_vq:
369 vring_del_virtqueue(vq);
370unmap:
371 vpdev->hw_ops->iounmap(vpdev, vdev->vr[index]);
372 return ERR_PTR(err);
373}
374
375static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
376 struct virtqueue *vqs[],
377 vq_callback_t *callbacks[],
378 const char * const names[], const bool *ctx,
379 struct irq_affinity *desc)
380{
381 struct _vop_vdev *vdev = to_vopvdev(dev);
382 struct vop_device *vpdev = vdev->vpdev;
383 struct mic_device_ctrl __iomem *dc = vdev->dc;
384 int i, err, retry;
385
386
387 if (nvqs > ioread8(&vdev->desc->num_vq))
388 return -ENOENT;
389
390 for (i = 0; i < nvqs; ++i) {
391 dev_dbg(_vop_dev(vdev), "%s: %d: %s\n",
392 __func__, i, names[i]);
393 vqs[i] = vop_find_vq(dev, i, callbacks[i], names[i],
394 ctx ? ctx[i] : false);
395 if (IS_ERR(vqs[i])) {
396 err = PTR_ERR(vqs[i]);
397 goto error;
398 }
399 }
400
401 iowrite8(1, &dc->used_address_updated);
402
403
404
405
406 vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
407 for (retry = 100; --retry;) {
408 if (!ioread8(&dc->used_address_updated))
409 break;
410 msleep(100);
411 };
412
413 dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry);
414 if (!retry) {
415 err = -ENODEV;
416 goto error;
417 }
418
419 return 0;
420error:
421 vop_del_vqs(dev);
422 return err;
423}
424
425
426
427
428static struct virtio_config_ops vop_vq_config_ops = {
429 .get_features = vop_get_features,
430 .finalize_features = vop_finalize_features,
431 .get = vop_get,
432 .set = vop_set,
433 .get_status = vop_get_status,
434 .set_status = vop_set_status,
435 .reset = vop_reset,
436 .find_vqs = vop_find_vqs,
437 .del_vqs = vop_del_vqs,
438};
439
440static irqreturn_t vop_virtio_intr_handler(int irq, void *data)
441{
442 struct _vop_vdev *vdev = data;
443 struct vop_device *vpdev = vdev->vpdev;
444 struct virtqueue *vq;
445
446 vpdev->hw_ops->ack_interrupt(vpdev, vdev->h2c_vdev_db);
447 list_for_each_entry(vq, &vdev->vdev.vqs, list)
448 vring_interrupt(0, vq);
449
450 return IRQ_HANDLED;
451}
452
453static void vop_virtio_release_dev(struct device *_d)
454{
455
456
457
458
459}
460
461
462
463
464
465static int _vop_add_device(struct mic_device_desc __iomem *d,
466 unsigned int offset, struct vop_device *vpdev,
467 int dnode)
468{
469 struct _vop_vdev *vdev;
470 int ret;
471 u8 type = ioread8(&d->type);
472
473 vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
474 if (!vdev)
475 return -ENOMEM;
476
477 vdev->vpdev = vpdev;
478 vdev->vdev.dev.parent = &vpdev->dev;
479 vdev->vdev.dev.release = vop_virtio_release_dev;
480 vdev->vdev.id.device = type;
481 vdev->vdev.config = &vop_vq_config_ops;
482 vdev->desc = d;
483 vdev->dc = (void __iomem *)d + _vop_aligned_desc_size(d);
484 vdev->dnode = dnode;
485 vdev->vdev.priv = (void *)(u64)dnode;
486 init_completion(&vdev->reset_done);
487
488 vdev->h2c_vdev_db = vpdev->hw_ops->next_db(vpdev);
489 vdev->virtio_cookie = vpdev->hw_ops->request_irq(vpdev,
490 vop_virtio_intr_handler, "virtio intr",
491 vdev, vdev->h2c_vdev_db);
492 if (IS_ERR(vdev->virtio_cookie)) {
493 ret = PTR_ERR(vdev->virtio_cookie);
494 goto kfree;
495 }
496 iowrite8((u8)vdev->h2c_vdev_db, &vdev->dc->h2c_vdev_db);
497 vdev->c2h_vdev_db = ioread8(&vdev->dc->c2h_vdev_db);
498
499 ret = register_virtio_device(&vdev->vdev);
500 if (ret) {
501 dev_err(_vop_dev(vdev),
502 "Failed to register vop device %u type %u\n",
503 offset, type);
504 goto free_irq;
505 }
506 writeq((u64)vdev, &vdev->dc->vdev);
507 dev_dbg(_vop_dev(vdev), "%s: registered vop device %u type %u vdev %p\n",
508 __func__, offset, type, vdev);
509
510 return 0;
511
512free_irq:
513 vpdev->hw_ops->free_irq(vpdev, vdev->virtio_cookie, vdev);
514kfree:
515 kfree(vdev);
516 return ret;
517}
518
519
520
521
522static int vop_match_desc(struct device *dev, void *data)
523{
524 struct virtio_device *_dev = dev_to_virtio(dev);
525 struct _vop_vdev *vdev = to_vopvdev(_dev);
526
527 return vdev->desc == (void __iomem *)data;
528}
529
530static void _vop_handle_config_change(struct mic_device_desc __iomem *d,
531 unsigned int offset,
532 struct vop_device *vpdev)
533{
534 struct mic_device_ctrl __iomem *dc
535 = (void __iomem *)d + _vop_aligned_desc_size(d);
536 struct _vop_vdev *vdev = (struct _vop_vdev *)readq(&dc->vdev);
537
538 if (ioread8(&dc->config_change) != MIC_VIRTIO_PARAM_CONFIG_CHANGED)
539 return;
540
541 dev_dbg(&vpdev->dev, "%s %d\n", __func__, __LINE__);
542 virtio_config_changed(&vdev->vdev);
543 iowrite8(1, &dc->guest_ack);
544}
545
546
547
548
549
550static int _vop_remove_device(struct mic_device_desc __iomem *d,
551 unsigned int offset, struct vop_device *vpdev)
552{
553 struct mic_device_ctrl __iomem *dc
554 = (void __iomem *)d + _vop_aligned_desc_size(d);
555 struct _vop_vdev *vdev = (struct _vop_vdev *)readq(&dc->vdev);
556 u8 status;
557 int ret = -1;
558
559 if (ioread8(&dc->config_change) == MIC_VIRTIO_PARAM_DEV_REMOVE) {
560 dev_dbg(&vpdev->dev,
561 "%s %d config_change %d type %d vdev %p\n",
562 __func__, __LINE__,
563 ioread8(&dc->config_change), ioread8(&d->type), vdev);
564 status = ioread8(&d->status);
565 reinit_completion(&vdev->reset_done);
566 unregister_virtio_device(&vdev->vdev);
567 vpdev->hw_ops->free_irq(vpdev, vdev->virtio_cookie, vdev);
568 iowrite8(-1, &dc->h2c_vdev_db);
569 if (status & VIRTIO_CONFIG_S_DRIVER_OK)
570 wait_for_completion(&vdev->reset_done);
571 kfree(vdev);
572 iowrite8(1, &dc->guest_ack);
573 dev_dbg(&vpdev->dev, "%s %d guest_ack %d\n",
574 __func__, __LINE__, ioread8(&dc->guest_ack));
575 iowrite8(-1, &d->type);
576 ret = 0;
577 }
578 return ret;
579}
580
581#define REMOVE_DEVICES true
582
583static void _vop_scan_devices(void __iomem *dp, struct vop_device *vpdev,
584 bool remove, int dnode)
585{
586 s8 type;
587 unsigned int i;
588 struct mic_device_desc __iomem *d;
589 struct mic_device_ctrl __iomem *dc;
590 struct device *dev;
591 int ret;
592
593 for (i = sizeof(struct mic_bootparam);
594 i < MIC_DP_SIZE; i += _vop_total_desc_size(d)) {
595 d = dp + i;
596 dc = (void __iomem *)d + _vop_aligned_desc_size(d);
597
598
599
600
601
602 rmb();
603 type = ioread8(&d->type);
604
605
606 if (type == 0)
607 break;
608
609 if (type == -1)
610 continue;
611
612
613 dev = device_find_child(&vpdev->dev, (void __force *)d,
614 vop_match_desc);
615 if (dev) {
616 if (remove)
617 iowrite8(MIC_VIRTIO_PARAM_DEV_REMOVE,
618 &dc->config_change);
619 put_device(dev);
620 _vop_handle_config_change(d, i, vpdev);
621 ret = _vop_remove_device(d, i, vpdev);
622 if (remove) {
623 iowrite8(0, &dc->config_change);
624 iowrite8(0, &dc->guest_ack);
625 }
626 continue;
627 }
628
629
630 dev_dbg(&vpdev->dev, "%s %d Adding new virtio device %p\n",
631 __func__, __LINE__, d);
632 if (!remove)
633 _vop_add_device(d, i, vpdev, dnode);
634 }
635}
636
637static void vop_scan_devices(struct vop_info *vi,
638 struct vop_device *vpdev, bool remove)
639{
640 void __iomem *dp = vpdev->hw_ops->get_remote_dp(vpdev);
641
642 if (!dp)
643 return;
644 mutex_lock(&vi->vop_mutex);
645 _vop_scan_devices(dp, vpdev, remove, vpdev->dnode);
646 mutex_unlock(&vi->vop_mutex);
647}
648
649
650
651
652static void vop_hotplug_devices(struct work_struct *work)
653{
654 struct vop_info *vi = container_of(work, struct vop_info,
655 hotplug_work);
656
657 vop_scan_devices(vi, vi->vpdev, !REMOVE_DEVICES);
658}
659
660
661
662
663static irqreturn_t vop_extint_handler(int irq, void *data)
664{
665 struct vop_info *vi = data;
666 struct mic_bootparam __iomem *bp;
667 struct vop_device *vpdev = vi->vpdev;
668
669 bp = vpdev->hw_ops->get_remote_dp(vpdev);
670 dev_dbg(&vpdev->dev, "%s %d hotplug work\n",
671 __func__, __LINE__);
672 vpdev->hw_ops->ack_interrupt(vpdev, ioread8(&bp->h2c_config_db));
673 schedule_work(&vi->hotplug_work);
674 return IRQ_HANDLED;
675}
676
677static int vop_driver_probe(struct vop_device *vpdev)
678{
679 struct vop_info *vi;
680 int rc;
681
682 vi = kzalloc(sizeof(*vi), GFP_KERNEL);
683 if (!vi) {
684 rc = -ENOMEM;
685 goto exit;
686 }
687 dev_set_drvdata(&vpdev->dev, vi);
688 vi->vpdev = vpdev;
689
690 mutex_init(&vi->vop_mutex);
691 INIT_WORK(&vi->hotplug_work, vop_hotplug_devices);
692 if (vpdev->dnode) {
693 rc = vop_host_init(vi);
694 if (rc < 0)
695 goto free;
696 } else {
697 struct mic_bootparam __iomem *bootparam;
698
699 vop_scan_devices(vi, vpdev, !REMOVE_DEVICES);
700
701 vi->h2c_config_db = vpdev->hw_ops->next_db(vpdev);
702 vi->cookie = vpdev->hw_ops->request_irq(vpdev,
703 vop_extint_handler,
704 "virtio_config_intr",
705 vi, vi->h2c_config_db);
706 if (IS_ERR(vi->cookie)) {
707 rc = PTR_ERR(vi->cookie);
708 goto free;
709 }
710 bootparam = vpdev->hw_ops->get_remote_dp(vpdev);
711 iowrite8(vi->h2c_config_db, &bootparam->h2c_config_db);
712 }
713 vop_init_debugfs(vi);
714 return 0;
715free:
716 kfree(vi);
717exit:
718 return rc;
719}
720
721static void vop_driver_remove(struct vop_device *vpdev)
722{
723 struct vop_info *vi = dev_get_drvdata(&vpdev->dev);
724
725 if (vpdev->dnode) {
726 vop_host_uninit(vi);
727 } else {
728 struct mic_bootparam __iomem *bootparam =
729 vpdev->hw_ops->get_remote_dp(vpdev);
730 if (bootparam)
731 iowrite8(-1, &bootparam->h2c_config_db);
732 vpdev->hw_ops->free_irq(vpdev, vi->cookie, vi);
733 flush_work(&vi->hotplug_work);
734 vop_scan_devices(vi, vpdev, REMOVE_DEVICES);
735 }
736 vop_exit_debugfs(vi);
737 kfree(vi);
738}
739
740static struct vop_device_id id_table[] = {
741 { VOP_DEV_TRNSP, VOP_DEV_ANY_ID },
742 { 0 },
743};
744
745static struct vop_driver vop_driver = {
746 .driver.name = KBUILD_MODNAME,
747 .driver.owner = THIS_MODULE,
748 .id_table = id_table,
749 .probe = vop_driver_probe,
750 .remove = vop_driver_remove,
751};
752
753module_vop_driver(vop_driver);
754
755MODULE_DEVICE_TABLE(mbus, id_table);
756MODULE_AUTHOR("Intel Corporation");
757MODULE_DESCRIPTION("Intel(R) Virtio Over PCIe (VOP) driver");
758MODULE_LICENSE("GPL v2");
759